Custom Remote Reader

INTGeoServer provides a set of web services to retrieve geo data for web and standalone applications. GeoToolkit includesRemoteSeismicReader in the seismic module to get seismic data from the INTGeoServer. RemoteSeismicReader allows replacing "transport" protocol between server and client. This tutorial shows how to create a custom reader to read seismic data based on customization of transport protocol in RemoteSeismicReader. For demonstration purposes only, node.js is used on the server side in addition to some classes from GeoToolkit to read SEG-Y format. Using the same server-side implementation in production is not recommended, because it was created to show server and client communication only. This tutorial implements an API similar to INTGeoServer to read SEG-Y files located on the server.

# Create a Reader

First, provide a location of the server and a file to be visualized. Specify the version of protocol that is going to be called. In the current example, a local server and a custom protocol will be used. Install all npm packages and run the node server first.
To do this:

Go to the server folder inside the customremotereader folder and run npm install. Check to ensure the latest LTS version of node.js is installed.
Run node server.js

To run server-side code, install the following prerequisites:

The code below creates a RemoteSeismicDataSource that returns information from the remote server about the specified seismic data source. It includes meta information and a collection of keys, which can be utilized to make a query. This server server implementation doesn't support keys. Call the method open() to receive information from the server. This call is asynchronous and one of two methods will be called in case of success or failure. Next, create a query from the server using the "select" method. In the current example, the "empty query" is used to request all traces. This method returns an instance of the RemoteSeismicReader to be provided as parameter for a visualization pipeline. The reader retrieves groups of traces from the server by requests from pipeline. It never loads all traces to memory. As a rule, it keeps some traces in the memory cache and retrieves groups of traces to be visible in the specified widget. This example specifies a protocol node in the version field, which is discussed in more detail later.

# Display Data

The next step is similar to the Remote Seismic Reader tutorial. Create a pipeline and widget to display data.

# Create a Custom Protocol

To provide a custom protocol of data communication between client and server, first implement geotoolkit/seismic/data/RemoteReaderDataProvider and register it in the
geotoolkit/seismic/data/RemoteReaderDataProviderRegistry. Register it by the name "node."
RemoteReaderDataProvider requests to implement several methods:

open - return information about data source from the server
queryTraces - make a query based on the selected keys and return information on how many traces are in the result set
readTraces - return a set of the requested traces from the server in the binary format.

import {RemoteReaderDataProviderRegistry} from '@int/geotoolkit/seismic/data/RemoteReaderDataProviderRegistry';
import {obfuscate} from '@int/geotoolkit/lib';
import {HttpClient} from '@int/geotoolkit/http/HttpClient';
import {RemoteReaderDataProvider} from '@int/geotoolkit/seismic/data/RemoteReaderDataProvider';
class NodeServerDataProvider extends RemoteReaderDataProvider {
    constructor (options) {
        super();
        this.options = options;
        this.http = HttpClient.getInstance().getHttp();
    }

    createInstance (options) {
        return new NodeServerDataProvider(options);
    }

    open (fileName) {
        return this.http.get('seismicdata/' + encodeURIComponent(fileName), {
            'responseType': 'json',
            'baseURL': this.options['host']
        }).then(function (response) {
            if (response['data'] && response['data']['version']) {
                return response['data'];
            }
            return Promise.reject('Server error');
        }, function (error) {
            return Promise.reject('Cannot connect to the server! Run node server.js!');
        });
    }

    queryTraces (fileName, query) {
        return this.http.get('seismicquery/' + encodeURIComponent(fileName), {
            'responseType': 'json',
            'baseURL': this.options['host']
        }).then(function (response) {
            if (response['data'] && response['data']['version']) {
                return response['data'];
            }
            return Promise.reject('Server error');
        }, function (error) {
            return Promise.reject('Cannot connect to the server! Run node server.js!');
        });
    }

    readTraces (fileName, options) {
        if (options && !Array.isArray(options['traceIndexes'])) {
            options['traceIndexes'] = [];
            for (let i = options['from']; i <= options['to']; ++i) {
                options['traceIndexes'].push(i);
            }
        }
        return this.http.request({
            'url': 'enumeratedtraces',
            'baseURL': this.options['host'],
            'method': 'POST',
            'responseType': 'arraybuffer',
            'headers': {
                'Content-Type': 'application/json'
            },
            'data': {
                'file': fileName,
                'byteOrder': options['byteOrder'],
                'query': options['query'],
                'data': {
                    'byteOrder': options['byteOrder'],
                    'samples': options['samples'],
                    'headers': options['headers'],
                    'traceIndexes': options['traceIndexes']
                }
            },
            'transformResponse': function (response) {
                return response['data'];
            }
        });
    }
}

obfuscate(NodeServerDataProvider);
RemoteReaderDataProviderRegistry.getInstance().register('node', new NodeServerDataProvider());

# Server Implementation

The server side is based on the node.js and express frameworks.

The file server.js contains a standard initialization for express. Enable CORS and load routes.

import express from 'express';
import methodOverride from 'method-override';
import bodyParser from 'body-parser';
import errorHandler from 'errorhandler';
import '@int/geotoolkit/environment.js';
import {isWorker} from './cluster.js';
import {routes} from './routes';

// If isWorker is true then this thread is a worker thread and should start working
if (isWorker) {
    global.consoleLog('===============================================================\n');
    global.consoleLog('> LOADING ... \n');
    startHTTPServer();
}
function startHTTPServer () {
    let app = express();
    const PORT = 3001;

    app.listen(PORT, () => {
        app.use(bodyParser.json());
        app.use(methodOverride());
        app.use(bodyParser.urlencoded({extended: true}));

        app.all('/*', function (req, res, next) {
            // CORS headers
            res.header('Access-Control-Allow-Origin', '*'); // restrict it to the required domain
            res.header('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS');
            // Set custom headers for CORS
            res.header('Access-Control-Allow-Headers', 'Content-type,Accept,X-Access-Token,X-Key');
            if (req.method === 'OPTIONS') {
                res.status(200).end();
            } else {
                next();
            }
        });
        // routes ======================================================================
        routes(app); // load our routes and pass in our app and fully configured passport

        // error handling middleware should be loaded after the loading the routes
        if (app.get('env') === 'development') {
            app.use(errorHandler());
        }
        console.log('Express server listening on port ' + PORT);
    });
}

# Routes

This example uses GeoToolkit.JS on the server-side as well as node.js. To do this, use jsdom and browser bridge to emulate a desktop browser.

Node.js doesn't support FileAPI of the usual browser. In this case, provide an imlemenation of the localfile support based on the fs API.

The servers process three routes: seismicdata, seismicquery, enumeratetraces.

import '@int/geotoolkit/environment.js';
import {NodeExport} from '@int/geotoolkit/scene/exports/NodeExport';
import {GeometryUtil} from '@int/geotoolkit/util/GeometryUtil';
import {Path} from '@int/geotoolkit/scene/shapes/Path';
import {Group} from '@int/geotoolkit/scene/Group';
import {Rect} from '@int/geotoolkit/util/Rect';
import {Transformation} from '@int/geotoolkit/util/Transformation';
import {NormalizationType} from '@int/geotoolkit/seismic/pipeline/NormalizationType';
import {SeismicPipeline} from '@int/geotoolkit/seismic/pipeline/SeismicPipeline';
import {StandardSegyFormat} from '@int/geotoolkit/seismic/data/StandardSegyFormat';
import {SegyReader} from '@int/geotoolkit/seismic/data/SegyReader';
import {LocalFile} from '@int/geotoolkit/seismic/data/LocalFile';
import {DataFormatType} from '@int/geotoolkit/seismic/data/DataFormatType';


let createCanvas = __require_for_vite_BGaudh.default || __require_for_vite_BGaudh.createCanvas;

let gCanvas = null;
let gSeismicDefaults = {
    'colormap': 'WhiteBlack',
    'width': 1024,
    'height': 1024
};
let Status = {
    OK: 200,
    Error: 500
};
let parseType = function (type) {
    switch (type) {
        case DataFormatType.UInt:
            return 'UInt';
        case DataFormatType.Short:
            return 'Short';
        case DataFormatType.Int:
            return 'Int';
        case DataFormatType.UShort:
            return 'UShort';
        case DataFormatType.Float:
            return 'Float';
        case DataFormatType.Double:
            return 'Double';
    }
    return 'Unknown';
};
let parseSize = function (type) {
    switch (type) {
        case DataFormatType.UInt:
        case DataFormatType.Int:
            return 4;
        case DataFormatType.Short:
        case DataFormatType.UShort:
            return 2;
        case DataFormatType.Float:
            return 4;
        case DataFormatType.Double:
            return 8;
    }
    return 0;
};
let getSeismicHeaderSize = function (reader) {
    let fields = reader.getTraceHeaderFields();
    let field, size = 0, fieldsize;
    for (let i = 0; i < fields.length; ++i) {
        field = fields[i];
        fieldsize = parseSize(field.getDataType());
        size += fieldsize;
    }
    return size;
};
let getSeismicFormat = function (reader) {
    let fields = reader.getTraceHeaderFields();
    let result = [], field, size = 0, fieldsize;
    for (let i = 0; i < fields.length; ++i) {
        field = fields[i];
        fieldsize = parseSize(field.getDataType());
        result.push({'name': field.getName(), 'id': i, 'type': parseType(field.getDataType()), 'size': fieldsize});
        size += fieldsize;
    }
    return {
        'fields': result,
        'size': size
    };
};
let openSeismicFile = function (fileName) {
    return new Promise(function (resolve, reject) {
        let file = new LocalFile(fileName);
        let reader = new SegyReader(file, new StandardSegyFormat(), -6, 0);
        reader.loadMetaData(function (reader) {
            resolve(reader);
        });
    });
};
let readStatistics = function (reader) {
    return new Promise(function (resolve, reject) {
        reader.readDataSetStatistics(function (reader, statistics) {
            resolve(statistics);
        });
    });
};
let writeFieldValue = function (buffer, value, type, offset, le) {
    switch (type) {
        case DataFormatType.UInt: {
            if (le) {
                buffer.writeUInt32LE(value, offset);
            } else {
                buffer.writeUInt32BE(value, offset);
            }
            break;
        }
        case DataFormatType.Int: {
            if (le) {
                buffer.writeInt32LE(value, offset);
            } else {
                buffer.writeInt32BE(value, offset);
            }
            break;
        }
        case DataFormatType.Short: {
            if (le) {
                buffer.writeInt16LE(value, offset, true);
            } else {
                buffer.writeInt16BE(value, offset, true);
            }
            break;
        }
        case DataFormatType.UShort: {
            if (le) {
                buffer.writeUInt16LE(value, offset);
            } else {
                buffer.writeUInt16BE(value, offset);
            }
            break;
        }
        case DataFormatType.Float: {
            if (le) {
                buffer.writeFloatLE(value, offset);
            } else {
                buffer.writeFloatBE(value, offset);
            }
            break;
        }
        case DataFormatType.Double: {
            if (le) {
                buffer.writeDoubleLE(value, offset);
            } else {
                buffer.writeDoubleBE(value, offset);
            }
            break;
        }
    }
};
let printTraceLE = function (reader, trace, buffer, offset) {
    let traceHeaderFields = reader.getTraceHeaderFields();
    let i;
    for (i = 0; i < traceHeaderFields.length; ++i) {
        let headerValue = trace.getHeader(i);
        let field = traceHeaderFields[i];
        let size = parseSize(field.getDataType());
        writeFieldValue(buffer, headerValue, field.getDataType(), offset, true);
        offset += size;
    }
    let samples = trace.getSamples();
    for (i = 0; i < reader.getNumberOfSamples(); ++i) {
        buffer.writeFloatLE(samples[i], offset);
        offset += 4;
    }
    return offset;
};
let printTraceBE = function (reader, trace, buffer, offset) {
    let traceHeaderFields = reader.getTraceHeaderFields();
    let i;
    for (i = 0; i < traceHeaderFields.length; ++i) {
        let headerValue = trace.getHeader(i);
        let field = traceHeaderFields[i];
        let size = parseSize(field.getDataType());
        writeFieldValue(buffer, headerValue, field.getDataType(), offset, false);
        offset += size;
    }
    let samples = trace.getSamples();
    for (i = 0; i < reader.getNumberOfSamples(); ++i) {
        buffer.writeFloatLE(samples[i], offset);
        offset += 4;
    }
    return offset;
};
let generateSeismicImage = function (options) {
    return new Promise(function (resolve, reject) {
        openSeismicFile(options['file']).then(function (reader) {
            readStatistics(reader).then(function (statistics) {
                let pipeline = new SeismicPipeline({
                    'name': 'ExportImage',
                    'reader': reader,
                    'statistics': statistics
                });
                pipeline.setOptions({
                    'transparency': false,
                    'plot': {
                        'type': {
                            'Wiggle': false,
                            'InterpolatedDensity': true
                        },
                        'decimationSpacing': 0,
                        'clippingFactor': 0
                    },
                    'normalization': {
                        'type': NormalizationType.RMS,
                        'scale': 0.4
                    },
                    'colors': {
                        'colorMap': options['colormap']
                    }
                });

                let model = reader.getModelLimits();
                let limitsToModelTransformation = Transformation
                    .getRectToRectInstance(model, options['modelInDevice'], false, false, false);
                let limits = limitsToModelTransformation.inverseTransformRect(options['tile']);
                let tracesModelSpace = limits.intersect(reader.getModelLimits());

                let canvaselement = options['canvas'];
                let deviceSpace = new Rect(
                    0, 0, canvaselement.width, canvaselement.height
                );
                pipeline.exportToImage(
                    tracesModelSpace,
                    canvaselement,
                    deviceSpace,
                    0,
                    0,
                    function () {
                        resolve(canvaselement);
                    }
                );
            }, function (error) {
                reject(error);
            });
        },
        function (error) {
            reject(error);
        }
        );
    });
};
export const routes = function (app) {
    // Gets a file information
    app.get('/seismicdata/:file', function (req, res) {
        openSeismicFile(req.params['file']).then(function (reader) {
            readStatistics(reader).then(function (statistics) {
                res.json({
                    'version': 1,
                    'displayName': req.params['file'],
                    'numberOfSamples': reader.getNumberOfSamples(),
                    'numberOfTraces': reader.getNumberOfTraces(),
                    'startValue': 0,
                    'sampleRate': reader.getSampleRate(),
                    'statistics': statistics,
                    'traceHeader': getSeismicFormat(reader)
                });
            }, function (error) {
                res.error(error);
            });
        },
        function (error) {
            res.error(error);
        }
        );
    });
    app.get('/seismicquery/:file', function (req, res) {
        openSeismicFile(req.params['file']).then(function (reader) {
            readStatistics(reader).then(function (statistics) {
                res.json({
                    'version': 1,
                    'numberOfSamples': reader.getNumberOfSamples(),
                    'numberOfTraces': reader.getNumberOfTraces(),
                    'startValue': 0,
                    'sampleRate': reader.getSampleRate(),
                    'statistics': statistics
                });
            }, function (error) {
                res.error(error);
            });
        },
        function (error) {
            res.error(error);
        }
        );
    });
    app.post('/enumeratedtraces', function (req, res) {
        let body = req.body;
        let byteOrder = body['byteOrder'] != null ? body['byteOrder'] : 'LITTLE_ENDIAN';
        let printTrace = byteOrder === 'LITTLE_ENDIAN' ? printTraceLE : printTraceBE;
        let data = body['data'];
        openSeismicFile(body['file']).then(function (reader) {
            let headerLength = getSeismicHeaderSize(reader);
            let length = (data['traceIndexes'].length * (reader.getNumberOfSamples() * 4 + headerLength));
            let arr = new Uint8Array(length);
            let buffer = Buffer.from(arr.buffer);
            res.writeHead(Status.OK, {
                'Content-Type': 'application/octet-stream', 'Content-Length': length
            });
            reader.select({
                'traceIndexes': data['traceIndexes'] != null ? data['traceIndexes'] : [],
                'headers': true,
                'samples': true
            }, function (context) {
                let offset = 0;
                context.foreach(function (id, section) {
                    for (let i = 0; i < section.getNumberOfTraces(); ++i) {
                        let trace = section.getTraceByIndex(i);
                        offset = printTrace(reader, trace, buffer, offset, headerLength);
                    }
                });
                res.end(buffer);
            });
        });
    });

    // Generate seismic image
    app.get('/seismicimage/:file', function (req, res) {
        let file = decodeURIComponent(req.params['file']);
        let x = +req.query['x'], y = +req.query['y'], cropWidth = +req.query['cropWidth'], cropHeight = +req.query['cropHeight'];
        let imgW = +req.query['imgW'], imgH = +req.query['imgH'];
        let tile = new Rect(x, y, x + cropWidth, y + cropHeight);
        let modelInDevice = new Rect(0, 0, imgW, imgH);
        let colormap = gSeismicDefaults['colormap'];
        let imageWidth = tile.getWidth();
        let imageHeight = tile.getHeight();
        if (gCanvas == null || gCanvas.width !== imageWidth || gCanvas.height !== imageHeight) {
            let canvas = createCanvas();
            canvas.height = imageHeight;
            canvas.width = imageWidth;
            gCanvas = canvas;
        }
        generateSeismicImage({
            'canvas': gCanvas,
            'file': file,
            'width': imageWidth,
            'height': imageHeight,
            'colormap': colormap,
            'modelInDevice': modelInDevice,
            'tile': tile,
            'errors': []
        }).then(function (png) {
            res.writeHead(200, {'Content-Type': 'image/png'});
            res.end(png.toBuffer()); // Send the file data to the browser.
        }, function (m) {
            res.status(Status.Error).send(m);
        });
    });
    // Generate dynamic velocity image
    app.get('/generatedynamicimage/:file', function (req, res) {
        let x = +req.query['x'], y = +req.query['y'], cropWidth = +req.query['cropWidth'], cropHeight = +req.query['cropHeight'];
        let lx = +req.query['lx'], ly = +req.query['ly'], lwidth = +req.query['lwidth'], lheight = +req.query['lheight'];
        let imgW = +req.query['imgW'], imgH = +req.query['imgH'];
        let tile = new Rect(x, y, x + cropWidth, y + cropHeight);
        let model = new Rect(lx, ly, lx + lwidth, ly + lheight);
        let modelInDevice = new Rect(0, 0, imgW, imgH);

        let limitsToModelTransformation = Transformation.getRectToRectInstance(model, modelInDevice, false, false, false);
        let limits = limitsToModelTransformation.inverseTransformRect(tile);

        let tileInModel = new Rect(limits);
        let tileInDeviceRect = new Rect(tile);
        // The full model
        let srcLimits = model;
        limitsToModelTransformation = Transformation.getRectToRectInstance(tileInModel, tileInDeviceRect, false, false, false);
        // generate scene graph
        let stepWidth = 1;
        let waveWidth = srcLimits.getWidth() / 5; // waveWidth => Math.PI;
        let getVelocityValue = function (x, velocity, snapToStep, step) {
            if (step == null) {
                step = stepWidth;
            }
            let xValue = x / waveWidth * Math.PI;
            if (snapToStep) {
                x = ((x / step) >> 0) * step;
            }
            velocity.x = x;
            velocity.y = 1000 + Math.sin(xValue) * 100;
            return velocity;
        };


        let buildSceneGraphD = function (srcLimitsD, stepD) {
            let sceneGraphD = new Group()
                .setModelLimits(srcLimitsD);

            let pathD = new Path()
                .setLineStyle({
                    'width': 1,
                    'color': 'red',
                    'pixelsnapmode': {'x': true, 'y': true}
                })
                .setFillStyle('rgba(255, 216, 0, 0.5)')
                .moveTo(srcLimitsD.getLeft(), srcLimitsD.getBottom());
            let velocityD = {
                x: srcLimitsD.getLeft(),
                y: 0
            };
            let iD;
            for (iD = srcLimitsD.getLeft(); iD < srcLimitsD.getRight(); iD += stepD) {
                if (velocityD.y !== 0) {
                    pathD.lineTo(velocityD.x + stepD, velocityD.y);
                }
                velocityD = getVelocityValue(iD, velocityD, true, stepD);
                pathD.lineTo(velocityD.x, velocityD.y);
            }
            pathD.lineTo(iD, srcLimitsD.getBottom())
                .lineTo(srcLimitsD.getLeft(), srcLimitsD.getBottom())
                .close();
            sceneGraphD.addChild(pathD);
            return sceneGraphD;
        };

        let sceneGraph = buildSceneGraphD(srcLimits, stepWidth);
        let surface = null;
        if (req.query['dynamic'] === true) {
            // build dynamic model
            let stepD = GeometryUtil.getVectorLengthInModel(1, 0, limitsToModelTransformation);
            let srcLimitsD = tileInModel.clone().inflate(stepD * 2);
            let sceneGraphD = buildSceneGraphD(srcLimitsD, stepD);
            surface = NodeExport
                .exportToSurface(sceneGraphD, tileInDeviceRect.getWidth(), tileInDeviceRect.getHeight(), false, false, tileInModel);
        } else {
            surface = NodeExport
                .exportToSurface(sceneGraph, tileInDeviceRect.getWidth(), tileInDeviceRect.getHeight(), false, false, tileInModel);
        }
        let canvas = surface.getCanvas();
        res.writeHead(200, {'Content-Type': 'image/png'});
        res.end(canvas.toBuffer());
    });
};

# Result

The following example uses RemoteSeismicReader to read data from the server. All traces are retrieved from the server and displayed with SeismicWidget. The canvas below shows a result of the visualization. Steps to set up the Custom Remote Reader follow.

import { Plot } from "@int/geotoolkit/plot/Plot.ts";
import { SeismicWidget } from "@int/geotoolkit/seismic/widgets/SeismicWidget.ts";
import { SeismicColors } from "@int/geotoolkit/seismic/util/SeismicColors.ts";
import { NormalizationType } from "@int/geotoolkit/seismic/pipeline/NormalizationType.ts";
import { SeismicPipeline } from "@int/geotoolkit/seismic/pipeline/SeismicPipeline.ts";
import { RemoteSeismicDataSource } from "@int/geotoolkit/seismic/data/RemoteSeismicDataSource.ts";
import "/src/code/Seismic/Readers/CustomRemoteReader/nodeServerDataProvider.ts";
const createReader = function(onready, onfailure) {
  const host = "http://localhost:3001/";
  const data = new RemoteSeismicDataSource({
    "host": host,
    "file": "data/section.segy",
    "version": "node"
  });
  data.open(
    () => {
      data.select({}, (reader) => {
        onready(reader);
      });
    },
    (err) => {
      onfailure(err);
    }
  );
};
const createPipeline = function(reader) {
  const pipeline = new SeismicPipeline({
    "name": "Seismic",
    "reader": reader,
    "statistics": reader.getStatistics()
  });
  pipeline.setOptions({
    "normalization": {
      "type": NormalizationType.RMS,
      "scale": 0.4
    },
    "plot": {
      "type": {
        "wiggle": false,
        "interpolateddensity": true
      },
      "decimationspacing": 5
    },
    "colors": {
      "colormap": SeismicColors.getDefault().createNamedColorMap("RedWhiteBlack")
    }
  });
  return pipeline;
};
function createScene(canvas, onError) {
  const widget = new SeismicWidget({
    "colorbar": {
      "axis": {
        "tickgenerator": {
          "edge": {
            "tickvisible": false,
            "labelvisible": false
          }
        }
      }
    }
  });
  createReader((reader) => {
    const pipeline = createPipeline(reader);
    widget.setPipeline(pipeline);
    widget.setOptions({
      "layouttype": "inside",
      "statusbar": {
        "visible": false
      }
    });
    const headerFields = pipeline.getReader().getTraceHeaderFields();
    let headerInfo;
    for (let i = 0; i < headerFields.length; i++) {
      const header = headerFields[i];
      if (header.getName() === "SRCX") {
        widget.setTraceHeaderVisible(header, true);
      } else if (header.getName() === "TraceNumber") {
        widget.setTraceHeaderVisible(header, false);
      }
      headerInfo = widget.getTraceHeaderAxis(header);
      if (headerInfo) {
        headerInfo["label"].getTextStyle().setColor("#6b6b6b");
      }
    }
  }, (err) => {
    onError(true);
  });
  return new Plot({
    "canvaselement": canvas,
    "root": widget
  });
}
export { createScene };

createScene(document.querySelector('[ref="plot"]'), this.inputServerWarningDialog);