{"templateId":"markdown","sharedDataIds":{"sidebar":"sidebar-guides/sidebars.yaml"},"props":{"metadata":{"markdoc":{"tagList":[]},"type":"markdown"},"seo":{"title":"Table of Contents","description":"Accelerate E&P application development and protect your innovation by consuming our Data and Domain APIs / Platform APIs.","lang":"en-US","meta":[{"name":"robots","content":"noindex"}],"llmstxt":{"hide":true,"excludeFiles":[]}},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"table-of-contents","__idx":0},"children":["Table of Contents ",{"$$mdtype":"Tag","name":"a","attributes":{"name":"TOC"},"children":[]}]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#overview"},"children":["Overview"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#dataModel"},"children":["Status Data Model"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#howToPublishStatusEvents"},"children":["How to publish status and dataset details events"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#publishDirectlyInQueue"},"children":["Publish messages directly in message queue"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#publishUsingRestEndpoints"},"children":["Publish messages using endpoints"]}]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#sampleMessageContract"},"children":["Sample of status and dataset details message contracts"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#stagesAndStatuses"},"children":["Supported Stages and Statuses"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#RestrictingFieldSize"},"children":["Restricting Field Size"]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"overview","__idx":1},"children":["Overview ",{"$$mdtype":"Tag","name":"a","attributes":{"name":"overview"},"children":[]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Global Status monitoring is a mechanism that tracks the status of data journey/dataflows on the data platform."," ","This mechanism helps to track the status of files, data, and records ingested through the File Service, Storage API, and specific DOMS until it is consumed by dependent services."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/gsmflowdiagram.9d4eb858f12e46f9b7295375a61110745ced116406439fb2f398b9fafac68ee9.cff9a875.png","alt":"GSM Flowchart"},"children":[]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Every stage publishes status events to the message queue. Some stages, like WKS_SYNC, INGESTOR_SYNC, STORAGE_SYNC. etc., publish two statuses per record. The first status is for “In Progress” messages, and the second status is for a “SUCCESS” or “FAILED” message."," ","From there on, the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["Status Collector"]}," picks up all status events and normalizes them to store in persistent storage for future reference. Then the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["Status Processor"]}," provides an API to query and check the status of past datasets."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The core services of Global Status Monitoring are the Status Collector and Status Processor."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"status-processor-service","__idx":2},"children":["Status Processor Service"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Status Processor Service provides APIs that allow users to monitor the status of files, data, and records ingested through the File Service, Storage API, and specific DOMS until it is consumed by dependent services."," ","The status indicates whether the dataflow has finished or not and if it is ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["Successful"]}," or ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["Failed"]},"."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For more detailed information, follow ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/solutions/data-workspace/tutorial/status-processor-service"},"children":["Status Processor tutorial"]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"status-publisher-service","__idx":3},"children":["Status Publisher Service"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The Status Publisher Service provide APIs for publishing dataSet details and status messages to ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["statuschangedtopic"]},"."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For more detailed information, follow ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/solutions/data-workspace/tutorial/status-publisher-service"},"children":["Status Publisher tutorial"]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"status-data-model","__idx":4},"children":["Status Data Model ",{"$$mdtype":"Tag","name":"a","attributes":{"name":"dataModel"},"children":[]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Data Model properties help users  search for status with multiple or specific properties. Every request is tracked through a specific dataSetId or its associated correlationId."," ","correlationId is a unique id that tracks the status of a request at various stages of the Data Platform."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The Status Data Model has multiple tables for tracking the stages of dataflow."]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["DataSet Details"]}," - Dataset pertains to any data, such as file, collection of files, etc."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Status"]}," - Holds the status of a dataflow."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"how-to-publish-status-and-dataset-detail-events","__idx":5},"children":["How to publish status and dataset detail events ",{"$$mdtype":"Tag","name":"a","attributes":{"name":"howToPublishStatusEvents"},"children":[]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["There are two ways to publish status and dataset details in a message queue."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"1--publish-messages-directly-in-message-queue","__idx":6},"children":["1.  Publish messages directly in message queue. ",{"$$mdtype":"Tag","name":"a","attributes":{"name":"publishDirectlyInQueue"},"children":[]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["To publish status and dataset details directly to a message queue for a JAVA application, follow the steps below:"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Add the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["os core common"]}," library as a dependency"]}," - There are models, classes, and interfaces defined in the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["core common lib"]}," from Azure."," ",{"$$mdtype":"Tag","name":"br","attributes":{},"children":[]}," Ensure that you select the correct version of the library which includes the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["StatusDetails"]}," and ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["DatasetDetails"]}," classes, 0.13.0 or higher.",{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"core-common-library-contents-for-gsm","__idx":7},"children":["Core Common Library contents for GSM"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Models"]}," - ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["StatusDetails"]}," and ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["DatasetDetails"]}," - Use these two models to publish status and dataset details."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Utility"]}," - ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["AttributesBuilder"]}," - Creates an attributes map which is a required method of ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["IEventPublisher"]}," to publish status or dataset details."," ",{"$$mdtype":"Tag","name":"br","attributes":{},"children":[]}," The attributes map consists of the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["slb partition id"]}," and ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["correlation id"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Publisher Interface"]}," - ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["IEventPublisher"]}," - This is the interface that the cloud provider must implement to produce status and dataset details."," ",{"$$mdtype":"Tag","name":"br","attributes":{},"children":[]}," It contains a method which accepts the Message array and Attributes map. Message is an interface that is implemented by both Status and Dataset Details."]}]}]}]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"  <dependency>\n    <groupId>org.opengroup.osdu</groupId>\n    <artifactId>os-core-common</artifactId>\n    <version>0.13.0</version>\n  </dependency>\n"},"children":[]},{"$$mdtype":"Tag","name":"ol","attributes":{"start":2},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Add ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["core-lib-azure"]}," library as dependency"]}," - provides a publisher facade to publish messages to the ServiceBus, version 0.13.0 or higher."]}]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"  <dependency>\n    <groupId>org.opengroup.osdu</groupId>\n    <artifactId>core-lib-azure</artifactId>\n    <version>0.13.0</version>\n  </dependency>\n"},"children":[]},{"$$mdtype":"Tag","name":"ol","attributes":{"start":3},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Publisher to identify all scenarios to publish Status/Dataset Details"]}," - You should determine all possible scenarios in which either Status or Dataset Details need to be published."," ",{"$$mdtype":"Tag","name":"br","attributes":{},"children":[]}," A service can publish multiple sets of both Status and Dataset Details."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Cloud Implementation to publish Status/Dataset Details"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Provide an implementation of the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["IEventPublisher"]}," interface from the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["os core common"]}," library. The ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["publish"]}," method of ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["IEventPublisher"]}," must be implemented to publish an event in ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["statuschangedtopic"]}," that accepts:",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Array of type Message:  ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["Message"]}," is an interface implemented by both ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["StatusDetails"]}," and ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["DatasetDetails"]}," classes. The ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["Message"]}," array can be either Status or Dataset Details. For an example , refer to section ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["Sample of status and dataset details message"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Map of key-value pair attributes: Key and value are of type string. Examples: \"correlation-id\": \"1235345\" and \"slb-partition-id\": \"user defined value\"."]}]}]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["PublisherInfo"]}," to provide needed information for publishing:"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["batch - An array of messages, such as Status or DataSet Details."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["serviceBusTopicName - The name of the ServiceBus topic that is ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["statuschangedtopic"]},"."]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Call method ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["publishMessage"]}," from ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["MessagePublisher"]}," class that accepts:"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["DpsHeaders - Must contain \"correlation-id\" and \"slb-partition-id\"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["PublisherInfo - The ServiceBus topic name and data to publish."]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Enable ServiceBus publishing by providing the following configuration"]}," - ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["azure.serviceBus.enabled = true"]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"sample-implementation-of-ieventpublisher-to-publish-messages-to-azure","__idx":8},"children":["Sample implementation of ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["IEventPublisher"]}," to publish messages to Azure:"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"  public class StatusEventPublisher implements IEventPublisher {\n\n  private final MessagePublisher messagePublisher;\n  private final ServiceBusConfig serviceBusConfig;\n  private final DpsHeaders dpsHeaders;\n  private final JaxRsDpsLog log;\n\n  @Override\n  public void publish(Message[] messages, Map<String, String> attributesMap) throws CoreException {\n      PublisherInfo publisherInfo = PublisherInfo.builder()\n              .batch(messages)\n              .serviceBusTopicName(serviceBusConfig.getServiceBusTopic())\n              .build();\n      messagePublisher.publishMessage(dpsHeaders, publisherInfo);\n      log.info(\"Status event generated successfully\");\n  }\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"sample-implementation-on-statuspublisher","__idx":9},"children":["Sample implementation on StatusPublisher"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"  public class StatusPublisher {\n      private static final String FAILED_TO_PUBLISH_STATUS = \"Failed to publish status \";\n      private static final String PUBLISH_STATUS_STARTED = \"Publish Status started\";\n   \n      private final IEventPublisher statusEventPublisher;\n      private final AttributesBuilder attributesBuilder;\n      private final JaxRsDpsLog log;\n   \n      private void publish(List<StatusDetails> statusDetailsList) {\n          try {\n              log.info(PUBLISH_STATUS_STARTED);\n              Map<String, String> attributesMap = attributesBuilder.createAttributesMap();\n               \n              statusEventPublisher.publish(statusDetailsList.toArray(new StatusDetails[0]), attributesMap);\n          } catch (CoreException e) {\n              log.warning(FAILED_TO_PUBLISH_STATUS + e.getMessage());\n              throw new ApplicationException(FAILED_TO_PUBLISH_STATUS, e);\n          }\n      }\n  }\n"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Note: We have an Azure implementation of Global Status Monitoring. Services that are not part of the OSDU AKS cluster have to use /status and /datasetDetails endpoints of the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["Status Processor"]}," service. The ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["Status Processor"]}," service publishs status and dataset details in ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["statuschangedtopic"]},"."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"2-publish-messages-using-endpoints","__idx":10},"children":["2. Publish messages using endpoints ",{"$$mdtype":"Tag","name":"a","attributes":{"name":"publishUsingRestEndpoints"},"children":[]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["There are two endpoints to publish status messages in the queue. These messages can be used by applications outside the OSDU cluster if they want to use GSM to track the status."]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Publish Status"]}," - POST /status endpoint"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Publish Dataset Details"]}," - POST /dataset-details endpoint"]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Follow ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/solutions/data-workspace/tutorial/status-publisher-service"},"children":["Status Publisher tutorial"]}," for more information related to publish endpoints."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"sample-of-status-and-dataset-details-message-contracts","__idx":11},"children":["Sample of status and dataset details message contracts ",{"$$mdtype":"Tag","name":"a","attributes":{"name":"sampleMessageContract"},"children":[]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["There are two kinds of status messages, DataSet Details and Status, but they are published into the same Azure ServiceBus ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["statuschangedtopic"]},"."]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["DataSet Details"]}]}]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"{\n  \"message\":{\n    \"data\":[\n      {\n        \"kind\": \"datasetDetails\",\n        \"properties\": {\n          \"correlationId\": \"1cc462e9-9a39-48b9-84c3-9d53b18b8089\",\n          \"datasetId\": \"opendes:dataset--File.Generic:332b55d5-0cf2-4474-a3e5-d3cff0c5746a\",\n          \"datasetVersionId\": \"opendes:dataset--File.Generic:332b55d5-0cf2-4474-a3e5-d3cff0c5746a:23423432\",\n          \"datasetType\": \"FILE\",\n          \"recordCount\": 1,\n          \"timestamp\": 1625221800\n        }\n      }\n    ],\n    \"account-id\":\"opendes\",\n    \"slb-partition-id\":\"opendes\",\n    \"correlation-id\":\"3f41d31b-862f-40ce-9749-ee26a3f714f2\"\n  }\n}\n"},"children":[]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Status"]}]}]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"{\n  \"message\":{\n    \"data\":[\n      {\n        \"kind\": \"status\",\n        \"properties\": {\n          \"correlationId\": \"1cc462e9-9a39-48b9-84c3-9d53b18b8089\",\n          \"recordId\": \"opendes:wellbore:osdudemo-ATVMxMDEzTVMxMDQ\",\n          \"recordIdVersion\": \"opendes:wellbore:osdudemo-ATVMxMDEzTVMxMDQ:23423432\",\n          \"stage\": \"STORAGE_SYNC\",\n          \"status\": \"FAILED\",\n          \"message\": \"acl is not valid\",\n          \"errorCode\": 400,\n          \"userEmail\": \"test@email.com\",\n          \"timestamp\": 1625221800\n        }\n      }\n    ],\n    \"account-id\":\"opendes\",\n    \"slb-partition-id\":\"opendes\",\n    \"correlation-id\":\"3f41d31b-862f-40ce-9749-ee26a3f714f2\"\n  }\n}\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"supported-stages-and-statuses","__idx":12},"children":["Supported Stages and Statuses ",{"$$mdtype":"Tag","name":"a","attributes":{"name":"stagesAndStatuses"},"children":[]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"stages-and-services-mapping","__idx":13},"children":["Stages and Services Mapping"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The stage shows the current activity of the record. For example if the record is in WKS transformation, this will be at the WKS_SYNC stage."]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Stage"},"children":["Stage"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Service"},"children":["Service"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Usage"},"children":["Usage"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["DATASET_SYNC"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["File Service, Dataset"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Emits all status events related to File Metadata Record creation."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["INGESTOR"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["All Ingestors for e.g., CSV, LAS/DLIS/Document, CI-doc. This status is a DAG level status"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Ingestors publish events under this stage when they receive calls from the Workflow Service and when they are finished with all the steps of the DAG."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The INGESTOR SUBMITTED status is emitted by the Workflow Service and the INGESTOR IN_PROGRESS and COMPLETE or FAILED statuess are emitted by the Specific Ingestor."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Here the SUCCESS status means that all of the steps of the DAG were invoked and completed. It does not mean that raw records were successfully created in the data platform."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["INGESTOR_SYNC"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["All Ingestors for e.g., CSV, LAS/DLIS/Document, CI-doc. ",{"$$mdtype":"Tag","name":"br","attributes":{},"children":[]}," This status is a record level status"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Ingestors publish status events under this stage about the records that will be stored after parsing the file. ",{"$$mdtype":"Tag","name":"br","attributes":{},"children":[]}," This status is different from the one above because the INGESTOR stage is a DAG level status, but INGESTOR_SYNC is record level status. If there are 10 records in the CSV to ingest, then the CSV is supposed to generate 10 status messages."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["WKS_SYNC"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["All those services that create WKS source records in the Data Platform, for e.g.:"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Services that are involved in standardizing the records publish status under this stage."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["* WKS Transformation Service"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The WKS service generates WKS records for the raw records ingested by CSV ingestion, so the WKS service publishes status events for each record it transforms."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["* Document Enrichment Service"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Document Ingestion records are standardized within the Document Enrichment Service, so that the service publishes status for each document record."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["* Wellbore DDMS"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["LAS/DLIS Ingestors directly create WKS records by hitting the Wellbore DDMS, and in this case, the Wellbore DDMS publishes status events for each record it creates."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["DATA_MASTER_SYNC"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["WKE Service"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The WKE Creation Service publishes status events under this stage for its progress about WKE record creation."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["STORAGE_SYNC"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Storage Service"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The Storage service publishes status events under this stage for every record it processes."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["ES_SYNC"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Indexer Service"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The indexer publishes status events under this stage for each record, whether it is indexed or not."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["QUALITY_CHECK"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["DQM"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Services involved with the checking and correcting the quality of the data."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["RELATIONSHIP_SYNC"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Relationship & Lineage Service"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Services that maintain the relationship between various records of the data platform in the graph DB."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["SF_SYNC"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Spotfire"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Services that are responsible for indexing records in Spotfire."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["IMAGE_GENERATION"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Image Generator Service"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The Document Insight Image Generator Service Publishes IN_PROGRESS/SUCCESS/FAILED status events under this stage for each page."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["PAGE_CLASSIFICATION"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Page Classification Service"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The Document Insight Page Classification Service Publishes SUCCESS/FAILED status event under this stage for each page when the page-classification is successful or fails."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["PAGE_TEXT_GENERATION"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Image OCR Service"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The Document Insight Image OCR Service Publishes SUCCESS/FAILED status event under this stage for each page when the image-ocr is successful or fails."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["DOCUMENT_INSIGHTS_GENERATION"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Insight Generator Service"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The Document Insight Insight Generator Service Publishes SUCCESS/FAILED status event under this stage for the particular document when insight generation is successful or fails."]}]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"supported-statuses","__idx":14},"children":["Supported Statuses"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The status of the record shows if it was processed as ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["Successful"]}," or ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["Failed"]},"."]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Status"},"children":["Status"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["SUBMITTED"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["SUCCESS"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["FAILED"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["IN_PROGRESS"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["SKIPPED"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["PARTIAL_SUCCESS"]}]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"restricting-field-size","__idx":15},"children":["Restricting Field Size ",{"$$mdtype":"Tag","name":"a","attributes":{"name":"RestrictingFieldSize"},"children":[]}]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Status Fields Max Allowed Size"]}]}]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Field Name"},"children":["Field Name"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Size"},"children":["Size"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Description"},"children":["Description"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["message"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["1000 chars"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The Status Message that contains information useful for the user, such as the reason for failure, reason for success, etc. ",{"$$mdtype":"Tag","name":"br","attributes":{},"children":[]},"The maximum value length of ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["message"]}," is 1000 chars. When the length is exceeded, the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["message"]}," is trimmed to the 1000th chararcter and stored."]}]}]}]}]}]},"headings":[{"value":"Table of Contents","id":"table-of-contents","depth":2},{"value":"Overview","id":"overview","depth":2},{"value":"Status Processor Service","id":"status-processor-service","depth":3},{"value":"Status Publisher Service","id":"status-publisher-service","depth":3},{"value":"Status Data Model","id":"status-data-model","depth":2},{"value":"How to publish status and dataset detail events","id":"how-to-publish-status-and-dataset-detail-events","depth":2},{"value":"1.  Publish messages directly in message queue.","id":"1--publish-messages-directly-in-message-queue","depth":3},{"value":"Core Common Library contents for GSM","id":"core-common-library-contents-for-gsm","depth":4},{"value":"Sample implementation of IEventPublisher to publish messages to Azure:","id":"sample-implementation-of-ieventpublisher-to-publish-messages-to-azure","depth":4},{"value":"Sample implementation on StatusPublisher","id":"sample-implementation-on-statuspublisher","depth":4},{"value":"2. Publish messages using endpoints","id":"2-publish-messages-using-endpoints","depth":3},{"value":"Sample of status and dataset details message contracts","id":"sample-of-status-and-dataset-details-message-contracts","depth":2},{"value":"Supported Stages and Statuses","id":"supported-stages-and-statuses","depth":2},{"value":"Stages and Services Mapping","id":"stages-and-services-mapping","depth":3},{"value":"Supported Statuses","id":"supported-statuses","depth":3},{"value":"Restricting Field Size","id":"restricting-field-size","depth":2}],"frontmatter":{"seo":{"title":"Table of Contents"}},"lastModified":"2025-10-30T09:33:28.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/solutions/data-workspace/tutorial/global-status-monitoring","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}