This class represents a generic cloud storage dataset. More...

#include <SDGenericDataset.h>

Public Member Functions
	SDGenericDataset (SDManager *sdmanager, const std::string &sdfilename, const bool log=false)
	Constructor. More...

	SDGenericDataset (SDManager *sdmanager, const std::string &sdfilename, const std::string &filetype, const bool log=false)
	Constructor with dataset type. More...

	~SDGenericDataset ()
	Destructor. More...

	SDGenericDataset (SDGenericDataset &&rhs) noexcept
	Move Constructor. More...

SDGenericDataset &	operator= (SDGenericDataset &&rhs) noexcept
	Move Assignment Operator. More...

	SDGenericDataset (const SDGenericDataset &rhs)
	Copy Constructor. More...

SDGenericDataset &	operator= (const SDGenericDataset &rhs)
	Copy Assignment Operator. More...

void	open (const SDDatasetDisposition disposition)
	Opens the dataset whose name was specified the constructor. More...

void	open (const SDDatasetDisposition disposition, const std::string &legaltag)
	Opens the dataset whose name was specified the constructor. More...

void	open (const SDDatasetDisposition disposition, const std::string &legaltag, const std::string &wid)
	Opens the dataset whose name was specified the constructor. More...

void	open (SDDatasetDisposition disposition, const std::unordered_map< std::string, std::string > &args)
	Main method to open a seismic store dataset. More...

void	open (const dataset::ParamsOpen &params)
	Opens the dataset whose name was specified in the constructor. More...

void	close ()
	Close an opened dataset. More...

std::string	getGcsUrl () const
	Deprecated. More...

std::string	getStorageUrl () const
	Get storage url. More...

std::string	getType () const
	Gets the dataset type. More...

std::string	getLegalTag () const
	Gets the legal-tag. More...

bool	blockExists (int blocknum) const
	Determines whether a block of data exists. More...

bool	blockExists (const std::string &) const
	Determines whether a block of data exists. More...

void	writeBlock (int blocknum, const char *data, std::size_t len, bool check_and_overwrite=false)
	Writes a block of data. More...

void	readBlock (int blocknum, char *data, size_t offset, size_t numBytes)
	Read a block of data. More...

void	readBlock (int blocknum, char *data, size_t &len)
	Read a block of data. More...

void	readBlock (int blocknum, char **data, size_t &len)
	Read a block of data. More...

void	writeBlock (const std::string &blockName, const char *data, std::size_t len, bool check_and_overwrite=false)
	Writes a block of data. More...

void	readBlock (const std::string &blockName, char *data, size_t offset, size_t numBytes)
	Read a block of data. More...

void	readBlock (const std::string &blockName, char *data, size_t &len)
	Read a block of data. More...

void	readBlock (const std::string &blockName, char **data, size_t &len)
	Read a block of data. More...

long long	getBlockSize (int blocknum)
	Gets the size of a block. More...

long long	getBlockSize (const std::string &blockName)
	Gets the size of a block. More...

std::vector< long long >	getBlocksSize (const std::vector< std::string > &blockNames)
	Deprecated. More...

std::vector< long long >	getBlockSizes (const std::vector< std::string > &blockNames)
	Gets the sizes of a list of blocks. More...

void	deleteBlock (const std::string &blockName)
	Deletes a given block. More...

void	deleteBlock (const std::string &blockName, bool skipMetadataUpdate)
	Deletes a given block. More...

uint64_t	getBlockNum ()
	Gets the number of blocks. More...

uint64_t	getSize ()
	Get the dataset size. More...

void	setBlockNum (uint64_t blockNum)
	sets the number of a block More...

void	setSize (uint64_t size)
	Sets the dataset size. More...

bool	checkReadAccess () const
	Checks if the user have read access permission on current dataset. More...

bool	checkWriteAccess () const
	Checks if the user have write access permission on current dataset. More...

bool	checkDeleteAccess () const
	Checks if the user have delete access permission on current dataset. More...

void	setMetaData (const std::string &metadata) const
	Adds user-defined metadata to the dataset. More...

std::string	getMetaData () const
	Gets the user-defined metadata of the current dataset. More...

std::string	getFileMetaData () const
	Gets the user-defined file metadata of the current dataset. More...

void	setSeismicMeta (const std::string &seismicmeta)
	Adds seismic metadata to the dataset. More...

std::string	getSeismicMeta () const
	Gets the seismic metadata from the dataset. More...

std::string	getCTag ()
	Gets the dataset coherency tag. More...

std::string	getConsistencyID ()
	Gets the locking session id. More...

int	getConsistencyCounter ()
	Gets process number that is locking the dataset. More...

bool	checkCTag ()
	Check dataset coherency tag. More...

void	refreshDatsetMetadata ()
	Deprecated. More...

void	refreshDatasetMetadata ()
	Refresh the dataset metadata. More...

SDGenericDatasetBasicObjectIterator	getIterator () const
	Gets the iterator info. More...

SDGenericDatasetBasicObjectIterator	getIterator (bool recursive) const
	Gets the iterator info. More...

void	flush ()
	Flushes the file metadata. More...

bool	exist () const
	Deprecated. More...

bool	exists () const
	Check if the dataset exists. More...

std::string	getSerializedReadOnlyAccessor () const
	Generate a serialized Read Only Dataset Accessor. More...

void	upload (const std::string &filepath, const std::string &legaltag="") const
	Upload a file into a single object. More...

void	setTags (const std::vector< std::string > &tags) const
	Add dataset tags. More...

std::vector< std::string >	getTags () const
	Get dataset tags. More...

std::string	getCreatedDate () const
	Get dataset created date. More...

std::string	getStatus () const
	Get dataset status. More...

void	setHttpContext (const HttpContext *context=nullptr)
	Set the underlying HTTP connection configuration. More...

void	setHttpContext (const HttpContext *context, HttpConnectionLink link)
	Set the underlying HTTP connection configuration. More...

void	setExponentialRetryBackoffPolicy (const ExponentialRetryBackoffPolicy *policy, HttpConnectionLink link=HttpConnectionLink::ANY)
	Set the exponential retry back-off policy. More...

std::string	getSerializedContext ()
	Return the dataset open context. More...

void	setReadonlyMode (bool readonly)
	Set the readonly mode for a dataset. More...

bool	getReadonlyMode () const
	Get the readonly mode status for a dataset. More...

std::vector< std::string >	getHosts ()
	Get host list for a dataset. More...

bool	updateStorageTier (const std::string &storageTier) const

bool	setStorageTier (const std::string &tier)
	set the storage tier from user More...

std::string	getStorageTier ()
	Get the storage tier from user. More...

Friends
class	SDGenericDatasetTest

Detailed Description

This class represents a generic cloud storage dataset.

This class is a generic dataset class to read/write cloud storage objects as sequential(0...n-1) file blocks.

Constructor & Destructor Documentation

◆ SDGenericDataset() [1/4]

seismicdrive::SDGenericDataset::SDGenericDataset	(	SDManager *	sdmanager,
		const std::string &	sdfilename,
		const bool	log = `false`
	)

Constructor.

Parameters

sdmanager	is the seismic drive service manager
sdfilename	name of the dataset. It must be in the form /seismic-drive/[tenant_name]/[subproject_name]/[path]/[dataset_name]. The field [path] rapresent the dataset hierarchy and it's optional. example sd://fatty-acid/sproj01/a/b/c/dataset_01
log	is a boolean value that enable the debug logger (false by default)

◆ SDGenericDataset() [2/4]

seismicdrive::SDGenericDataset::SDGenericDataset	(	SDManager *	sdmanager,
		const std::string &	sdfilename,
		const std::string &	filetype,
		const bool	log = `false`
	)

Constructor with dataset type.

Parameters

sdmanager	is the seismic drive service manager
sdfilename	name of the dataset. It must be in the form /seismic-drive/[tenant_name]/[subproject_name]/[path]/[dataset_name]. The field [path] rapresent the dataset hierarchy and it's optional. example sd://fatty-acid/sproj01/a/b/c/dataset_01
filetype	is the dataset type (ex. dio, bolt, txt, segy... )
log	is a boolean value that enable the debug logger (false by default)

◆ ~SDGenericDataset()

seismicdrive::SDGenericDataset::~SDGenericDataset ( )

Destructor.

If the file is opened for writing (not READ_ONLY), it will be closed.

◆ SDGenericDataset() [3/4]

seismicdrive::SDGenericDataset::SDGenericDataset ( SDGenericDataset && rhs )

noexcept

Move Constructor.

This constructor moves the SDGenericDataset object from the source to the new object and destroy the source object.

◆ SDGenericDataset() [4/4]

seismicdrive::SDGenericDataset::SDGenericDataset ( const SDGenericDataset & rhs )

Copy Constructor.

This constructor makes a copy of the SDGenericDataset object from the source to the new object without modifying the source object.

Member Function Documentation

◆ blockExists() [1/2]

bool seismicdrive::SDGenericDataset::blockExists ( int blocknum ) const

Determines whether a block of data exists.

This method determines whether a block of data (specified with a block name) exists.

Parameters

blocknum is the number of the block

Returns: true if the block exists, otherwise false.

◆ blockExists() [2/2]

bool seismicdrive::SDGenericDataset::blockExists ( const std::string & ) const

Determines whether a block of data exists.

This method determines whether a block of data (specified with a block name) exists.

Parameters

blockName is the name of the block

Returns: true if the block exists, otherwise false.

◆ checkCTag()

bool seismicdrive::SDGenericDataset::checkCTag ( )

Check dataset coherency tag.

Returns true if the cached tag returned by getCTag() still matches what is stored on the server.

Returns false if the dataset has been updated. In this case, all the other cached data for this dataset must be assumed stale as well.

Raises an exception if the dataset has been deleted.

NOTES: This method is somewhat expensive as it sends a request to the server.

A dataset that has been deleted and re-created with the same name is considered "updated", not "missing", and will not raise an exception.

The cached c-tag will not be updated by this function.

A dataset has its c-tag updated

when opened with SDDatasetDisposition::OVERWRITE
when closed, if opened for writing, unless pedantic working mode was disabled and a wtag (consistency ID) was used to open
when its tags are modified
when its metadata is modified
when its subproject's metadata is modified

The c-tag does not change when data blocks are written.

Usage example:

SDManager sdmanager;
SDGenericDataset dataset(&sdmanager, "sd://tenant/subproject/data");
dataset.open(SDDatasetDisposition::READ_ONLY);
bool isCtagOk = dataset.checkCTag();
dataset.close();

Returns: true if the dataset c-tag matches the cached one, false if not.

◆ checkDeleteAccess()

bool seismicdrive::SDGenericDataset::checkDeleteAccess ( ) const

Checks if the user have delete access permission on current dataset.

This method checks if the user have delete access permission on current dataset

Returns: true if the user has delete access, and false otherwise.

◆ checkReadAccess()

bool seismicdrive::SDGenericDataset::checkReadAccess ( ) const

Checks if the user have read access permission on current dataset.

This method checks if the user have read access permission on current dataset

Returns: true if the user has read access, and false otherwise.

◆ checkWriteAccess()

bool seismicdrive::SDGenericDataset::checkWriteAccess ( ) const

Checks if the user have write access permission on current dataset.

This method checks if the user have write access permission on current dataset

Returns: true if the user has write access, and false otherwise.

◆ close()

void seismicdrive::SDGenericDataset::close ( )

Close an opened dataset.

This method closes a dataset and marks it as closed. File must be opened.

Exceptions

error::dataset::Locked

if opened in CREATE, OVERWRITE or READ_WRITE mode and Seismic Store received an HTTP Error 423 from service this may happen if

an HTTP call was retried after a 502 timeout ( proxy returned failure early ) either access to the lock cache failed ( with a short TTL ) or the lock cache contains an inconsistent data ( with a long TTL ) at which point user may want to retry after short TTL or SDUtil::unlock() and retry
user is attempting to open a dataset legitimately locked by some other application

The caller must decide whether it's appropriate to retry. error::dataset::Locked should contain enough information to make that decision.

◆ deleteBlock() [1/2]

void seismicdrive::SDGenericDataset::deleteBlock ( const std::string & blockName )

Deletes a given block.

This method is deletes a block given its name. If the object does not exist, this method does nothing.

Parameters

blockName is the name of the block to delete

◆ deleteBlock() [2/2]

void seismicdrive::SDGenericDataset::deleteBlock	(	const std::string &	blockName,
		bool	skipMetadataUpdate
	)

Deletes a given block.

This method is deletes a block given its name. If the object does not exist, this method does nothing.

Parameters

blockName	is the name of the block to delete
skipMetadataUpdate	if true, the dataset metadata won't be updated after deletion

◆ exist()

bool seismicdrive::SDGenericDataset::exist ( ) const

Deprecated.

Use exists() instead

◆ exists()

bool seismicdrive::SDGenericDataset::exists ( ) const

Check if the dataset exists.

Checks if the dataset exists in cloud storage

Returns: true if the dataset exists, otherwise false

◆ flush()

void seismicdrive::SDGenericDataset::flush ( )

Flushes the file metadata.

This method flushes the metadata of the object into the dataset.

◆ getBlockNum()

uint64_t seismicdrive::SDGenericDataset::getBlockNum ( )

Gets the number of blocks.

This method is an accessor used to get the block number.

Returns: the number of blocks.

◆ getBlockSize() [1/2]

long long seismicdrive::SDGenericDataset::getBlockSize ( int blocknum )

Gets the size of a block.

This method is an accessor used to get the size of a block in bytes given the block number.

Parameters

blocknum should be in the range 0..getBlockNum()-1

Returns: the size of the block number blocknum -1 if the block doesn't exist

◆ getBlockSize() [2/2]

long long seismicdrive::SDGenericDataset::getBlockSize ( const std::string & blockName )

Gets the size of a block.

This method is an accessor used to get the size of a block in bytes given the block name.

Parameters

blockName is the name of the block

Returns: the size of the block named blockName -1 if the block doesn't exist

◆ getBlockSizes()

std::vector<long long> seismicdrive::SDGenericDataset::getBlockSizes ( const std::vector< std::string > & blockNames )

Gets the sizes of a list of blocks.

This method is an accessor used to get the size of a list of blocks in bytes given a vector of their names.

Parameters

blockNames is the list of blocks name

Returns: the sizes of the blocks named in the vector blockNames

◆ getBlocksSize()

std::vector<long long> seismicdrive::SDGenericDataset::getBlocksSize ( const std::vector< std::string > & blockNames )

Deprecated.

Use getBlockSizes() instead

◆ getConsistencyCounter()

int seismicdrive::SDGenericDataset::getConsistencyCounter ( )

Gets process number that is locking the dataset.

This method is an accessor used to get the process number that is locking the current dataset.

Returns: the consistency process counter

◆ getConsistencyID()

std::string seismicdrive::SDGenericDataset::getConsistencyID ( )

Gets the locking session id.

This method is an accessor used to get the locking session ID of the current dataset.

Returns: the locking id

◆ getCreatedDate()

std::string seismicdrive::SDGenericDataset::getCreatedDate ( ) const

Get dataset created date.

This method can be used to retrieve the dataset creation date.

usage example:

SDManager sdmanager;
SDGenericDataset dataset(&sdmanager, "sd://tenant/subproject/data");
dataset.open(SDDatasetDisposition::READ_ONLY);
auto date = dataset.getCreatedDate();
dataset.close();

Returns: The dataset created date

◆ getCTag()

std::string seismicdrive::SDGenericDataset::getCTag ( )

Gets the dataset coherency tag.

Returns the CTag for the dataset that is cached on the client side, i.e. the value the tag has on the last explicit call to open() or refreshDatasetMetadata(). There is no check to see whether the cache is stale. The method has very low cost.

usage example:

SDManager sdmanager;
SDGenericDataset dataset(&sdmanager, "sd://tenant/subproject/data");
dataset.open(SDDatasetDisposition::READ_ONLY);
auto ctag = dataset.getCTag();
dataset.close();

Returns: the dataset coherency tag

◆ getFileMetaData()

std::string seismicdrive::SDGenericDataset::getFileMetaData ( ) const

Gets the user-defined file metadata of the current dataset.

This method is an accessor used to get the user-defined file metadata from the current dataset

Returns: the stringify json file metadata of the current dataset

◆ getGcsUrl()

std::string seismicdrive::SDGenericDataset::getGcsUrl ( ) const

Deprecated.

Use getStorageUrl() instead

◆ getHosts()

std::vector<std::string> seismicdrive::SDGenericDataset::getHosts ( )

Get host list for a dataset.

Returns a list of hosts potentially used to interact with this dataset.

NOTE: The dataset must be open.

Returns: A list of hosts

◆ getIterator() [1/2]

SDGenericDatasetBasicObjectIterator seismicdrive::SDGenericDataset::getIterator ( ) const

Gets the iterator info.

This method gets the info from the dataset's SDGenericDatasetBasicObjectIterator object.

Returns: the Generic Dataset Basic Object Iterator

◆ getIterator() [2/2]

SDGenericDatasetBasicObjectIterator seismicdrive::SDGenericDataset::getIterator ( bool recursive ) const

Gets the iterator info.

This method gets the info from the dataset's SDGenericDatasetBasicObjectIterator object.

Parameters

recursive determines the depth of the search

Returns: the Generic Dataset Basic Object Iterator

◆ getLegalTag()

std::string seismicdrive::SDGenericDataset::getLegalTag ( ) const

Gets the legal-tag.

This method gets the legal-tag.

Returns: the legal-tag.

◆ getMetaData()

std::string seismicdrive::SDGenericDataset::getMetaData ( ) const

Gets the user-defined metadata of the current dataset.

This method is an accessor used to get the user-defined metadata from the current dataset

Returns: the stringify json metadata of the current dataset

◆ getReadonlyMode()

bool seismicdrive::SDGenericDataset::getReadonlyMode ( ) const

Get the readonly mode status for a dataset.

This method return the dataset readonly mode status as boolean

usage example:

SDManager sdmanager;
SDGenericDataset dataset01(&sdmanager, "sd://tenant/subproject/data");
dataset01.open(seistore::SDDatasetDisposition::READ_WRITE); // any disposition
auto readonly = dataset01.getReadonlyMode();
dataset01.close();

Returns: The dataset readonly mode status

◆ getSeismicMeta()

std::string seismicdrive::SDGenericDataset::getSeismicMeta ( ) const

Gets the seismic metadata from the dataset.

This method can be used to retrieve the seismic metadata field from the dataset's JSON

usage example:

SDManager sdmanager; SDGenericDataset dataset(&sdmanager, "sd://tenant/subproject/data"); dataset.open(SDDatasetDisposition::READ_ONLY); auto smeta = dataset.getSeismicMeta(); dataset.close()

Returns: the stringify json seismic metadata

◆ getSerializedContext()

std::string seismicdrive::SDGenericDataset::getSerializedContext ( )

Return the dataset open context.

This method returns the serialized context of an open read-only dataset. The serialized context can be injected as query parameter in the dataset URI to load the serialized context instead of generating a new one.

usage example:

SDManager sdmanager;
SDGenericDataset dataset01(&sdmanager, "sd://tenant/subproject/data");
dataset01.open(seistore::SDDatasetDisposition::READ_ONLY);
context = dataset01.getSerializedContext();
dataset01.close();
SDGenericDataset dataset02(&sdmanager, "sd://tenant/subproject/data?context=" + context);
// this will load the context (bypass seistore calls)
dataset02.open(seistore::SDDatasetDisposition::READ_ONLY);
dataset02.readBlock("0", const_cast<char*>(res.c_str()), 0, 4);
dataset02.close();

◆ getSerializedReadOnlyAccessor()

std::string seismicdrive::SDGenericDataset::getSerializedReadOnlyAccessor ( ) const

Generate a serialized Read Only Dataset Accessor.

This method generates read only accessor for the current dataset.

Returns: the serialized ReadOnly Dataset Accessor

◆ getSize()

uint64_t seismicdrive::SDGenericDataset::getSize ( )

Get the dataset size.

This method gets the dataset size, which is the total size of all blocks in the dataset.

Returns: the dataset size.

◆ getStatus()

std::string seismicdrive::SDGenericDataset::getStatus ( ) const

Get dataset status.

This method can be used to retrieve the dataset status (if set)

usage example:

SDManager sdmanager;
SDGenericDataset dataset(&sdmanager, "sd://tenant/subproject/data");
dataset.open(SDDatasetDisposition::READ_ONLY);
auto status = dataset.getStatus();
dataset.close();

Returns: The dataset created date

◆ getStorageTier()

std::string seismicdrive::SDGenericDataset::getStorageTier ( )

Get the storage tier from user.

This method will return storage tier, and only works for existing datasets.

Returns: A string representing the storage tier.

◆ getStorageUrl()

std::string seismicdrive::SDGenericDataset::getStorageUrl ( ) const

Get storage url.

Returns the storage url of this dataset

Returns: storage url

◆ getTags()

std::vector<std::string> seismicdrive::SDGenericDataset::getTags ( ) const

Get dataset tags.

This method can be used to retrieve the list of seismic store dataset tags.

usage example:

SDManager sdmanager;
SDGenericDataset dataset(&sdmanager, "sd://tenant/subproject/data");
dataset.open(SDDatasetDisposition::READ_ONLY);
auto tags = dataset.getTags();
dataset.close();

Returns: The list of seismic store dataset tags

◆ getType()

std::string seismicdrive::SDGenericDataset::getType ( ) const

Gets the dataset type.

This method gets the dataset type.

Returns: the dataset type.

◆ open() [1/5]

void seismicdrive::SDGenericDataset::open ( const SDDatasetDisposition disposition )

Opens the dataset whose name was specified the constructor.

This method is the most basic open function. It opens a dataset with a given open mode.

usage example:

SDManager sdmanager;
SDGenericDataset dataset(&sdmanager, "sd://tenant/subproject/data");
dataset.open(SDDatasetDisposition::CREATE);
dataset.close();

Parameters

disposition open mode (ignored if opened from a serialized context)

◆ open() [2/5]

void seismicdrive::SDGenericDataset::open	(	const SDDatasetDisposition	disposition,
		const std::string &	legaltag
	)

Opens the dataset whose name was specified the constructor.

This method allows opening a dataset with a given open mode and legaltag.

usage example:

SDManager sdmanager;
SDGenericDataset dataset(&sdmanager, "sd://tenant/subproject/data");
dataset.open(SDDatasetDisposition::CREATE, { {api::json::Constants::kLegalTagLabel, "legal-tag"});
dataset.close();

Parameters

disposition	open mode
legaltag	the dataset legaltag

Exceptions

error::dataset::Locked

if opened in CREATE, OVERWRITE or READ_WRITE mode and Seismic Store received an HTTP Error 423 from service this may happen if

an HTTP call was retried after a 502 timeout ( proxy returned failure early ) either access to the lock cache failed ( with a short TTL ) or the lock cache contains an inconsistent data ( with a long TTL ) at which point user may want to retry after short TTL or SDUtil::unlock() and retry
user is attempting to open a dataset legitimately locked by some other application

The caller must decide whether it's appropriate to retry. error::dataset::Locked should contain enough information to make that decision.

more realistic usage example :

SDManager sdmanger;
std::string path =  "sd://tenant/subproject/data" ;
SDGenericDataset dataset( &sdmanager, path);
auto create = [&dataset](){
    dataset.open( SDDatasetDisposition::CREATE,
              {
                     {api::json::Constants::kLegalTagLabel, "legal-tag"}
                     {api::json::Constants::kPedantic, api::json::KDisableProperty}
              }
     );
 }
 auto unlock = [&path]() { SDUtil::unlock( path ); }
 std::vector<std::string> additionalTags = computeMoreTags();
 auto tag = [&dataset, &additionalTags](){ dataset.setTags( additionalTags ); }
 auto close = [&dataset](){ dataset.close();}
 auto retryOn423 = []( std::function<void()> fn, std::function<void()> unlockFn ){
      // declare and fill some backoff object.
      // this is really crude
      ExponentialRetryBackoffPolicy policy;
      policy.enabled=true;
      policy.maxRetries = 10;
      for( int i = 0; i < policy.maxRetries; ++i ){
           try {
               fn();
               return;
           } catch( error::dataset::Locked const & pLocked ){
                switch( pLocked.reason() ){
                   case error::dataset::Locked::Reason::CANNOT_LOCK:
                   case error::dataset::Locked::Reason::CANNOT_UNLOCK:
                      {
                           // sleep for TTL if accurate ?
                           // or just truncated exponential backoff based on policy ?
                           //std::uint32_t ttlSeconds = pLocked.get_estimated_lock_lease_seconds();
                           // adapt backoff ?
                           continue;
                      }
                      break;
                   case error::dataset::Locked::Reason::WRITE_LOCKED:
                   case error::dataset::Locked::Reason::READ_LOCKED:
                      {
                          // TTL clearly unpalatable.
                          // if it is a mistake force an unlock.
                          unlockFn();
                          // some kind of truncated exponential backoff based on policy
                          continue;
                      }
                      break;
                   case error::dataset::Locked::Reason::UNKNOWN:
                   default:
                      throw;
                }
           }
      }
 };
 retryOn423( create, unlock );
 dataset.writeBlock(... )
 retryOn423(    tag, unlock );
 retryOn423(  close, unlock );

◆ open() [3/5]

void seismicdrive::SDGenericDataset::open	(	const SDDatasetDisposition	disposition,
		const std::string &	legaltag,
		const std::string &	wid
	)

Opens the dataset whose name was specified the constructor.

This method allows opening a dataset with a given open mode and legaltag that has already been locked for writing.

usage example:

SDManager sdmanager;
SDGenericDataset dataset(&sdmanager, "sd://tenant/subproject/data");
dataset.open(SDDatasetDisposition::CREATE, { {api::json::Constants::kLegalTagLabel, "legal-tag"}, {api::json::Constants::kWriteIdLabel, "wid"});
dataset.close();

Parameters

disposition	open mode
wid	is the write opening id. This must be specified if you want to open a dataset already locked for write.
legaltag	the dataset legaltag

Exceptions

error::dataset::Locked

if opened in CREATE, OVERWRITE or READ_WRITE mode and Seismic Store received an HTTP Error 423 from service this may happen if

an HTTP call was retried after a 502 timeout ( proxy returned failure early ) either access to the lock cache failed ( with a short TTL ) or the lock cache contains an inconsistent data ( with a long TTL ) at which point user may want to retry after short TTL or SDUtil::unlock() and retry
user is attempting to open a dataset legitimately locked by some other application

The caller must decide whether it's appropriate to retry. error::dataset::Locked should contain enough information to make that decision.

◆ open() [4/5]

void seismicdrive::SDGenericDataset::open	(	SDDatasetDisposition	disposition,
		const std::unordered_map< std::string, std::string > &	args
	)

Main method to open a seismic store dataset.

This method opens a seismic data dataset in seismic store.

The method support different opening disposition modes:

CREATE: create a dataset and open it for write
OVERWRITE: create a dataset and open it for write overriding the existing one.
READ_WRITE: open an existing dataset for read and write.
READ_ONLY: open an existing dataset for read only.

The method accepts these optional opening arguments (defined in Constants.h);

kLegalTagLabel: the legal tag,
kWriteIdLabel: the open write session id -> open a slave process
kDDMSSeismicMetadataLabel: the seismic storage metadata.
kFlushManifest: define if the file-metadata* (manifest) have be saved along with the dataset metadata on close. = "enable" (default) -> save the file-metadata manifest on close. = "disable" -> do no save the file-metadata manifest on close.
KPedantic: define if enable or disable the pedantic mode to flush the file-metadata in a locking session. = "enable" (default) -> the file-metadata, if exist, will be saved. = "disable" -> the file-metadata, if exist, will be saved ONLY if the kWriteIdLabel is not set (not in a locking session)
KReadonlyOnClose: define if the dataset should be marked as readonly when the dataset is closed. = "enable" -> mark the dataset as readonly on close. = "disable" (default) -> do not mark the dataset as readonly on close. The file-metadata, in the SDGenericDataset accessor, is an object metadata associated to the dataset. It contains information on the number of objects that compose the dataset and the total size in byte: file-metadata = { type: "GENERIC", nobjects: <number_of_objects>, size: <dataset_size> }

These properties are collected at every block written and saved during the close operation. In the case of a "Master" node non-pedantic (see table below) the file-metadata is not collected at every block written, but is computed during the close by iterating on the saved object in the storage (high costs)

The file-metadata can be useful if dataset is saved as single object or as consecutive sequence like: 0 1 2 3 ..... N object This information is saved and, in some cases, computed during the close dataset operation. If not required, it is advisable to disable the file-metadata saving. This will improve performance by reducing the latencies while closing datasets.

The optional argument, KFlushManifest/KPedantic/KWriteLabel, can be tuned based on the consumer workflow. Below reported all possible combination with: saved = YES (save the file metadata) NO (do not save the file metadata) cost = number of call to perform to the SDMS service during the close

|-kFlushManifest-|-KPedantic-|-kWriteIdLabel-|-saved-|-cost-| |-ENABLE------—|-ENABLE-—| -SET-------—|-YES—|-1-—| |-ENABLE------—|-ENABLE-—| -UNSET-----—|-YES—|-1-—| |-ENABLE------—|-DISABLE—| -SET-------—|-NO-—|-0-—| |-ENABLE------—|-DISABLE—| -UNSET-----—|-YES—|-1-—| |-DISABLE-----—|-ENABLE-—| -SET-------—|-NO-—|-0-—| |-DISABLE-----—|-ENABLE-—| -UNSET-----—|-NO-—|-0-—| |-DISABLE-----—|-DISABLE—| -SET-------—|-NO-—|-0-—| |-DISABLE-----—|-DISABLE—| -UNSET-----—|-NO-—|-0— |

usage example:

SDManager sdmanager;
SDGenericDataset dataset(&sdmanager, "sd://tenant/subproject/data");
dataset.open(SDDatasetDisposition::CREATE, { {api::json::Constants::kLegalTagLabel, "legal-tag"}, {api::json::Constants::kWriteIdLabel, "wid"}, {api::json::Constants::kDDMSSeismicMetadataLabel, "{\"data":"value"}"} {api::json::Constants::kPedantic, "enable" || "disable"}, {api::json::Constants::kFlushManifest, "enable" || "disable"}, {api::json::Constants::KReadonlyOnClose, "enable" || "disable"} });
dataset.close();

Parameters

disposition	The opening disposition open mode
args	The list of optional arguments

Exceptions

error::dataset::Locked

if opened in CREATE, OVERWRITE or READ_WRITE mode and Seismic Store received an HTTP Error 423 from service this may happen if

an HTTP call was retried after a 502 timeout ( proxy returned failure early ) either access to the lock cache failed ( with a short TTL ) or the lock cache contains an inconsistent data ( with a long TTL ) at which point user may want to retry after short TTL or SDUtil::unlock() and retry
user is attempting to open a dataset legitimately locked by some other application

The caller must decide whether it's appropriate to retry. error::dataset::Locked should contain enough information to make that decision.

◆ open() [5/5]

void seismicdrive::SDGenericDataset::open ( const dataset::ParamsOpen & params )

Opens the dataset whose name was specified in the constructor.

Accepts structure containing parameters specific to this call

usage example:

SDManager manager;
SDGenericDataset dataset(&manager, "sd://tenant/subproject/data");
dataset.open(ParamsOpen{Params{}.withCorrelationId("abc-123")}.withDisposition(SDDatasetDisposition::READ_ONLY));
dataset.close();

Parameters

params dataset opening parameters

◆ operator=() [1/2]

SDGenericDataset& seismicdrive::SDGenericDataset::operator= ( SDGenericDataset && rhs )

noexcept

Move Assignment Operator.

Same functionality as the move constructor.

◆ operator=() [2/2]

SDGenericDataset& seismicdrive::SDGenericDataset::operator= ( const SDGenericDataset & rhs )

Copy Assignment Operator.

Same functionality as the copy constructor.

◆ readBlock() [1/6]

void seismicdrive::SDGenericDataset::readBlock	(	int	blocknum,
		char *	data,
		size_t	offset,
		size_t	numBytes
	)

Read a block of data.

This method reads a block of data specified by the block number. It expects the size of the data to read in bytes. This method does not check the crc32c.

Parameters

blocknum	should be in the range 0..getBlockNum()-1
data	is a buffer where read data will be saved (pre-allocated)
offset	is the starting reading offset
numBytes	is the size in byte of the data to read

◆ readBlock() [2/6]

void seismicdrive::SDGenericDataset::readBlock	(	int	blocknum,
		char *	data,
		size_t &	len
	)

Read a block of data.

This method reads a block of data specified by the block number. This method does check the crc32c.

Parameters

blocknum	should be in the range 0..getBlockNum()-1
data	is a buffer where read data will be saved (pre-allocated)
len	is the param where the reading len will be stored

◆ readBlock() [3/6]

void seismicdrive::SDGenericDataset::readBlock	(	int	blocknum,
		char **	data,
		size_t &	len
	)

Read a block of data.

This method reads a block of data specified by the block number. It will overwrite any pre-allocated buffers where the data read is to be stored. This method does check the crc32c. *

Parameters

blocknum	should be in the range 0..getBlockNum()-1
data	is a pointer to a buffer where read data will be saved (pre-allocated data will be lost if any)
len	is the param where the reading len will be stored

◆ readBlock() [4/6]

void seismicdrive::SDGenericDataset::readBlock	(	const std::string &	blockName,
		char *	data,
		size_t	offset,
		size_t	numBytes
	)

Read a block of data.

This method reads a block of data specified by the block name. It expects the size of the data to read in bytes. This method does not check the crc32c.

Parameters

blockName	is the name of the block to read
data	is a buffer where read data will be saved (pre-allocated)
offset	is the starting reading offset
numBytes	is the size in byte of the data to read

◆ readBlock() [5/6]

void seismicdrive::SDGenericDataset::readBlock	(	const std::string &	blockName,
		char *	data,
		size_t &	len
	)

Read a block of data.

This method reads a block of data specified by the block name. This method does check the crc32c.

Parameters

blockName	is the name of the block to read
data	is a buffer where read data will be saved (pre-allocated)
len	is the param where the reading len will be stored

◆ readBlock() [6/6]

void seismicdrive::SDGenericDataset::readBlock	(	const std::string &	blockName,
		char **	data,
		size_t &	len
	)

Read a block of data.

This method reads a block of data specified by the block number. It will overwrite any pre-allocated buffers where the data read is to be stored. This method does check the crc32c. *

Parameters

blockName	is the name of the block to read
data	is a pointer to a buffer where read data will be saved (pre-allocated data will be lost if any)
len	is the param where the reading len will be stored

◆ refreshDatasetMetadata()

void seismicdrive::SDGenericDataset::refreshDatasetMetadata ( )

Refresh the dataset metadata.

Unconditionally refresh the metadata from the SD server. You only need to call this if getCTag() returned false. Calling this method is semantically equivalent to closing the SDGenericDataset instance and opening a new one, but it might be more efficient. In particular, if a file has been deleted and re-created with the same name then neither refresh nor close/re-open will have a problem with that.

usage example:

SDManager sdmanager;
SDGenericDataset dataset(&sdmanager, "sd://tenant/subproject/data");
dataset.open(SDDatasetDisposition::READ_ONLY);
dataset.refreshDatasetMetadata();
dataset.close();

◆ refreshDatsetMetadata()

void seismicdrive::SDGenericDataset::refreshDatsetMetadata ( )

Deprecated.

Use refreshDatasetMetadata() instead

◆ setBlockNum()

void seismicdrive::SDGenericDataset::setBlockNum ( uint64_t blockNum )

sets the number of a block

This method sets the number of a block directly to the given number.

Parameters

blockNum is the number to be assigned to the block.

◆ setExponentialRetryBackoffPolicy()

void seismicdrive::SDGenericDataset::setExponentialRetryBackoffPolicy	(	const ExponentialRetryBackoffPolicy *	policy,
		HttpConnectionLink	link = `HttpConnectionLink::ANY`
	)

Set the exponential retry back-off policy.

This method configure the retry back applied policy

Parameters

policy	The exponential retry back-off policy settings.
HttpConnectionLink	The connection link where this settings must apply (ANY by default)

NOTE: This method is not thread-safe. Do not call it while more than one thread is accessing this dataset.

usage example:

ExponentialRetryBackoffPolicy policy;
policy.maxRetry = 5;
SDManager sdmanager;
SDGenericDataset dataset(&sdmanager, "sd://tenant/subproject/data");
dataset.setExponentialRetryBackoffPolicy(&policy, HttpConnectionLink::CLOUD_STORAGE);
// work with dataset
dataset.close();

◆ setHttpContext() [1/2]

void seismicdrive::SDGenericDataset::setHttpContext ( const HttpContext * context = nullptr )

Set the underlying HTTP connection configuration.

This method sets the underlying HTTP connection configuration

Parameters

context the underlying HTTP connection configuration

usage example:

HttpContext* storage_context = new HttpContext();
seismic_dms_context->timeoutSecs = 300; // 5 minutes
SDManager sdmanager;
SDGenericDataset dataset(&sdmanager, "sd://tenant/subproject/data");
dataset.setHttpContext(seismic_dms_context);
// work with dataset
dataset.close();

◆ setHttpContext() [2/2]

void seismicdrive::SDGenericDataset::setHttpContext	(	const HttpContext *	context,
		HttpConnectionLink	link
	)

Set the underlying HTTP connection configuration.

This method sets the underlying HTTP connection configuration

Parameters

context	the underlying HTTP connection configuration
HttpConnectionLink	The connection link where this settings must apply (ANY by default)

NOTE: This method is not thread-safe. Do not call it while more than one thread is accessing this dataset.

usage example:

HttpContext storage_context;
storage_context->timeoutSecs = 300; // 5 minutes
SDManager sdmanager;
SDGenericDataset dataset(&sdmanager, "sd://tenant/subproject/data");
dataset.setHttpContext(&storage_context, HttpConnectionLink::CLOUD_STORAGE);
// work with dataset
dataset.close();

◆ setMetaData()

void seismicdrive::SDGenericDataset::setMetaData ( const std::string & metadata ) const

Adds user-defined metadata to the dataset.

This method adds the user-defined metadata passed here to the current dataset.

Parameters

metadata is the json stringify metadata

Exceptions

error::dataset::Locked

◆ setReadonlyMode()

void seismicdrive::SDGenericDataset::setReadonlyMode ( bool readonly )

Set the readonly mode for a dataset.

This method set the dataset as readonly if the input parameter "readonly" is true

Exceptions

error::dataset::Locked usage example:

SDManager sdmanager;
SDGenericDataset dataset01(&sdmanager, "sd://tenant/subproject/data");
dataset01.open(seistore::SDDatasetDisposition::READ_WRITE); // or READ_ONLY
dataset01.setReadonlyMode(true);
dataset01.close();

◆ setSeismicMeta()

void seismicdrive::SDGenericDataset::setSeismicMeta ( const std::string & seismicmeta )

Adds seismic metadata to the dataset.

This method can be use to set the seismic metadata field in the dataset's JSON

usage example:

SDManager sdmanager; SDGenericDataset dataset(&sdmanager, "sd://tenant/subproject/data"); dataset.open(SDDatasetDisposition::READ_WRITE); dataset.setSeismicMeta("{\"description":""}"); dataset.close();

Parameters

seismicmeta is the json stringify seismic metadata. The schema for seismic meta can be found at https://developer.delfi.slb.com/solutions/dataecosystem/tutorials/storageservice

Exceptions

error::dataset::Error	Invalid JSON or seismicmeta field does not exist after patch request
error::dataset::Locked	( see above for retries )

◆ setSize()

void seismicdrive::SDGenericDataset::setSize ( uint64_t size )

Sets the dataset size.

This method sets the size of a dataset to the given number.

Parameters

size	is the size to be assigned to the dataset.

◆ setStorageTier()

bool seismicdrive::SDGenericDataset::setStorageTier ( const std::string & tier )

set the storage tier from user

This method will set storage tier base on user input.

Parameters

tier	is user provided storage tier.

◆ setTags()

void seismicdrive::SDGenericDataset::setTags ( const std::vector< std::string > & tags ) const

Add dataset tags.

This method can be used to set a list of tags to a seismic store dataset. Tags are just simple strings (not key/value pairs).

usage example:

SDManager sdmanager;
SDGenericDataset dataset(&sdmanager, "sd://tenant/subproject/data");
dataset.open(SDDatasetDisposition::READ_WRITE);
dataset.setTags({"TestTag01", "TestTag02", "TestTag03"});
dataset.close();

Parameters

tags	The list of tags to add to the opened seismic store dataset

Exceptions

error::dataset::Locked

if opened in CREATE, OVERWRITE or READ_WRITE mode and Seismic Store received an HTTP Error 423 from service this may happen if

an HTTP call was retried after a 502 timeout ( proxy returned failure early ) either access to the lock cache failed ( with a short TTL ) or the lock cache contains an inconsistent data ( with a long TTL ) at which point user may want to retry after short TTL or SDUtil::unlock() and retry
user is attempting to open a dataset legitimately locked by some other application

The caller must decide whether it's appropriate to retry. error::dataset::Locked should contain enough information to make that decision.

◆ updateStorageTier()

bool seismicdrive::SDGenericDataset::updateStorageTier ( const std::string & storageTier ) const

◆ upload()

void seismicdrive::SDGenericDataset::upload	(	const std::string &	filepath,
		const std::string &	legaltag = `""`
	)		const

Upload a file into a single object.

Uploads a file as a single object.

Parameters

filepath	is the absolute path of the file to upload
legaltag	the dataset legaltag

◆ writeBlock() [1/2]

void seismicdrive::SDGenericDataset::writeBlock	(	int	blocknum,
		const char *	data,
		std::size_t	len,
		bool	check_and_overwrite = `false`
	)

Writes a block of data.

This method writes a block of data (specified with a block number) with a specified length.

Parameters

blocknum	is the number of the block (keep it sequential 0...n-1)
data	is the block data to write
len	is the size in byte of the data to write
check_and_overwrite	is a flag that if set to true will force the system to check if a previous block with the same name exist

◆ writeBlock() [2/2]

void seismicdrive::SDGenericDataset::writeBlock	(	const std::string &	blockName,
		const char *	data,
		std::size_t	len,
		bool	check_and_overwrite = `false`
	)

Writes a block of data.

This method writes a block of data (specified with a block name) with a specified length.

Parameters

blockName	is the name of the block to write
data	is the block data to write
len	is the size in byte of the data to write
check_and_overwrite	is a flag that if set to true will force the system to check if a previous block with the same name exist

Friends And Related Function Documentation

◆ SDGenericDatasetTest

friend class SDGenericDatasetTest

friend

The documentation for this class was generated from the following file:

/osdu-sdapi-cpp/src/src/core/SDGenericDataset.h

Public Member Functions

Friends

Detailed Description

Constructor & Destructor Documentation

◆ SDGenericDataset() [1/4]

◆ SDGenericDataset() [2/4]

◆ ~SDGenericDataset()

◆ SDGenericDataset() [3/4]

◆ SDGenericDataset() [4/4]

Member Function Documentation

◆ blockExists() [1/2]

◆ blockExists() [2/2]

◆ checkCTag()

◆ checkDeleteAccess()

◆ checkReadAccess()

◆ checkWriteAccess()

◆ close()

◆ deleteBlock() [1/2]

◆ deleteBlock() [2/2]

◆ exist()

◆ exists()

◆ flush()

◆ getBlockNum()

◆ getBlockSize() [1/2]

◆ getBlockSize() [2/2]

◆ getBlockSizes()

◆ getBlocksSize()

◆ getConsistencyCounter()

◆ getConsistencyID()

◆ getCreatedDate()

◆ getCTag()

◆ getFileMetaData()

◆ getGcsUrl()

◆ getHosts()

◆ getIterator() [1/2]

◆ getIterator() [2/2]

◆ getLegalTag()

◆ getMetaData()

◆ getReadonlyMode()

◆ getSeismicMeta()

◆ getSerializedContext()

◆ getSerializedReadOnlyAccessor()

◆ getSize()

◆ getStatus()

◆ getStorageTier()

◆ getStorageUrl()

◆ getTags()

◆ getType()

◆ open() [1/5]

◆ open() [2/5]

◆ open() [3/5]

◆ open() [4/5]

◆ open() [5/5]

◆ operator=() [1/2]

◆ operator=() [2/2]

◆ readBlock() [1/6]

◆ readBlock() [2/6]

◆ readBlock() [3/6]

◆ readBlock() [4/6]

◆ readBlock() [5/6]

◆ readBlock() [6/6]

◆ refreshDatasetMetadata()

◆ refreshDatsetMetadata()

◆ setBlockNum()

◆ setExponentialRetryBackoffPolicy()

◆ setHttpContext() [1/2]

◆ setHttpContext() [2/2]

◆ setMetaData()

◆ setReadonlyMode()

◆ setSeismicMeta()

◆ setSize()

◆ setStorageTier()

◆ setTags()

◆ updateStorageTier()

◆ upload()

◆ writeBlock() [1/2]

◆ writeBlock() [2/2]

Friends And Related Function Documentation

◆ SDGenericDatasetTest