[ORCT-118] bookkeeping services bugfix synchronization tests

app/lib/alimonitor-services/AbstractServiceSynchronizer.js

@@ -23,6 +23,7 @@ const defaultServiceSynchronizerOptions = {
     forceStop: false,
     cacheRawResponse: process.env['RCT_DEV_USE_CACHE'] === 'false' ? false : true,
     useCacheJsonInsteadIfPresent: process.env['RCT_DEV_USE_CACHE_INSTEAD'] === 'true' ? true : false,
+    forceToUseOnlyCache: process.env['RCT_DEV_FORCE_CACHE_USAGE'] === 'true' ? true : false,
     batchSize: 4,
 };
 
@@ -128,32 +129,32 @@ class AbstractServiceSynchronizer {
         endpoint,
         metaDataHandler = null,
     ) {
-        try {
-            this.monitor = new PassCorrectnessMonitor(this.logger, this.errorsLoggingDepth);
-
-            const rawResponse = await this.getRawResponse(endpoint);
-            if (metaDataHandler) {
-                await metaDataHandler(rawResponse);
-            }
-
-            const data = this.processRawResponse(rawResponse)
+        this.monitor = new PassCorrectnessMonitor(this.logger, this.errorsLoggingDepth);
+        return await this.getRawResponse(endpoint)
+            .then(async (rawResponse) => {
+                if (metaDataHandler) {
+                    await metaDataHandler(rawResponse);
+                }
+                return rawResponse;
+            })
+            .then(async (rawResponse) => this.processRawResponse(rawResponse)
                 .filter((r) => {
                     const f = r && this.isDataUnitValid(r);
                     if (!f) {
                         this.monitor.handleOmitted();
                     }
                     return f;
-                });
-
-            await this.makeBatchedRequest(data, endpoint);
-
-            this.monitor.logResults();
-            return true;
-        } catch (fatalError) {
-            this.logger.error(fatalError.message + fatalError.stack);
-            await this.interrtuptSyncTask();
-            return false;
-        }
+                }))
+            .then(async (data) => await this.makeBatchedRequest(data, endpoint))
+            .then(() => {
+                this.monitor.logResults();
+                return true; // Passed without major errors
+            })
+            .catch(async (fatalError) => {
+                this.logger.error(`${fatalError.message} :: ${fatalError.stack}`);
+                this.interrtuptSyncTask();
+                return false; // Major error occurred
+            });
     }
 
     async makeBatchedRequest(data, endpoint) {
@@ -175,7 +176,7 @@ class AbstractServiceSynchronizer {
     }
 
     async getRawResponse(endpoint) {
-        if (this.useCacheJsonInsteadIfPresent && Cacher.isCached(this.name, endpoint)) {
+        if (this.useCacheJsonInsteadIfPresent && Cacher.isCached(this.name, endpoint) || this.forceToUseOnlyCache) {
             this.logger.info(`using cached json :: ${Cacher.cachedFilePath(this.name, endpoint)}`);
             return Cacher.getJsonSync(this.name, endpoint);
         }
@@ -200,10 +201,9 @@ class AbstractServiceSynchronizer {
         this.progressMonitor = new ProgressMonitor({ logger: this.logger.info.bind(this.logger), percentageStep: 0.25 });
         this.forceStop = false;
         return await this.sync(options)
-            .then(() => {
-                if (this.forceStop) {
-                    this.logger.info(`${this.name} forced to stop`);
-                }
+            .catch((fatalError) => {
+                this.logger.error(`${fatalError.message} :: ${fatalError.stack}`);
+                return false;
             });
     }
 
@@ -213,7 +213,7 @@ class AbstractServiceSynchronizer {
      * It will NOT interrupt any sequelize call being executed.
      * @return {void}
      */
-    async interrtuptSyncTask() {
+    interrtuptSyncTask() {
         this.forceStop = true;
     }
 

app/lib/alimonitor-services/BookkeepingService.js

@@ -135,6 +135,9 @@ class BookkeepingService extends AbstractServiceSynchronizer {
             .then(async ([beamType, _]) => await PeriodRepository.T.findOrCreate({
                 where: {
                     name: period.name,
+                },
+                default: {
+                    name: period.name,
                     year: period.year,
                     BeamTypeId: beamType.id,
                 },
@@ -181,15 +184,15 @@ class BookkeepingService extends AbstractServiceSynchronizer {
         const { page } = rawResponse['meta'];
         if (!page || !page['pageCount']) {
             this.logger.error(`No metadata found in Bookkeeping for the requested page: ${JSON.stringify(rawResponse)}`);
-            await this.interrtuptSyncTask();
+            this.interrtuptSyncTask();
             return;
         }
         this.metaStore['pageCount'] = page['pageCount'];
         this.metaStore['totalCount'] = page['totalCount'];
     }
 
     syncTraversStop(currentState) {
-        return this.isStopped() || currentState['page'] > this.metaStore['pageCount'];
+        return this.isStopped() || currentState['page'] >= this.metaStore['pageCount'];
     }
 
     nextState(state) {

app/lib/alimonitor-services/helpers/Cacher.js

@@ -60,6 +60,7 @@ class Cacher {
     static serviceCacheDir(synchronizerName) {
         return path.join(
             config.services.rawJsonCachePath,
+            'rawJson',
             synchronizerName,
         );
     }

app/lib/utils/http-utils.js

@@ -16,8 +16,6 @@
 const http = require('http');
 const https = require('https');
 const { Log } = require('@aliceo2/web-ui');
-const path = require('path');
-const fs = require('fs');
 
 const logger = new Log('Utils');
 
@@ -38,21 +36,20 @@ function checkClientType(endpoint) {
 function makeHttpRequestForJSON(url, opts, logger, onSuccess, onFailure) {
     url = new URL(url);
     return new Promise((resolve, reject) => {
-        let rawData = '';
+        let rawDataAccumulator = '';
         const req = checkClientType(url).request(url, opts, async (res) => {
             const { statusCode } = res;
             const contentType = res.headers['content-type'];
 
             let error;
             let redirect = false;
             if (statusCode == 302 || statusCode == 301) {
-                const mess = `Redirect. Status Code: ${statusCode}; red. to ${res.headers.location}`;
+                const mess = `Redirect. Status Code: ${statusCode}; red. to ${res.headers.location} from ${url.href}`;
                 if (opts.allowRedirects) {
                     redirect = true;
                     logger.warn(mess);
                     const nextHop = new URL(url.origin + res.headers.location);
                     nextHop.searchParams.set('res_path', 'json');
-                    logger.warn(`from ${url.href} to ${nextHop.href}`);
                     resolve(await makeHttpRequestForJSON(nextHop));
                 } else {
                     throw new Error(mess);
@@ -63,52 +60,23 @@ function makeHttpRequestForJSON(url, opts, logger, onSuccess, onFailure) {
                 error = new Error(`Invalid content-type. Expected application/json but received ${contentType}`);
             }
             if (error) {
-                logger.error(error.message);
-                res.resume();
-                return;
+                reject(error);
             }
 
             res.on('data', (chunk) => {
-                rawData += chunk;
-            });
-
-            req.on('error', (e) => {
-                logger.error(`ERROR httpGet: ${e}`);
-                if (onFailure) {
-                    onFailure(url, e);
-                }
-                reject(e);
+                rawDataAccumulator += chunk;
             });
 
             res.on('end', () => {
                 try {
                     if (!redirect) {
-                        /*
-                         * TMP incorrect format handling
-                         * if (/: *,/.test(rawData)) {
-                         *     rawData = rawData.replaceAll(/: *,/ig, ':"",');
-                         * }
-                         */
-
-                        const data = JSON.parse(rawData);
+                        const data = JSON.parse(rawDataAccumulator);
                         if (onSuccess) {
                             onSuccess(url, data);
                         }
                         resolve(data);
                     }
                 } catch (e) {
-                    logger.error(`${e.message} for endpoint: ${url.href}`);
-                    const fp = path.join(
-                        __dirname,
-                        '..',
-                        '..',
-                        '..',
-                        'database',
-                        'cache',
-                        'rawJson',
-                        'failing-endpoints.txt',
-                    );
-                    fs.appendFileSync(fp, `${url.href}\n     ${e.message}\n`);
                     if (onFailure) {
                         onFailure(url, e);
                     }
@@ -117,6 +85,13 @@ function makeHttpRequestForJSON(url, opts, logger, onSuccess, onFailure) {
             });
         });
 
+        req.on('error', (e) => {
+            if (onFailure) {
+                onFailure(url, e);
+            }
+            reject(e);
+        });
+
         req.end();
     });
 }

docker/test.env

@@ -78,3 +78,10 @@ RCT_EP_ML_MC_DET_PATH=${RCT_EP_ML_MC_DET_PATH:-/job_events.jsp?timesel=0&res_pat
 RCT_EP_ML_MC_TAG_PROT=$_DEF_ML_PROT
 RCT_EP_ML_MC_TAG_HOST=${RCT_EP_ML_MC_TAG_HOST:-$_DEF_ML_HOST}
 RCT_EP_ML_MC_TAG_PATH=${RCT_EP_ML_MC_TAG_PATH:-/MC/prodDetails.jsp?res_path=json}
+
+### cache
+RCT_DEV_USE_CACHE_INSTEAD=${RCT_DEV_USE_CACHE_INSTEAD:-true}
+RAW_JSON_CACHE_PATH=${RAW_JSON_CACHE_PATH:-/opt/RunConditionTable/4c3a64a02110a9f3ad7ad66f63f13fb84a2dc99c/}
+
+### other
+RCT_ERR_DEPTH=full

hooks/pre-commit/newlines.sh

@@ -7,12 +7,16 @@ done;
 FIND_EXCLUDE_PHRASE="$FIND_EXCLUDE_PHRASE -not -path './.git/*'"
 FIND_EXCLUDE_PHRASE="$FIND_EXCLUDE_PHRASE -not -path '*.png'"
 FIND_EXCLUDE_PHRASE="$FIND_EXCLUDE_PHRASE -not -path './app/public/styles/images/favicon/favicon.ico'" # it seems to be binary but for some reason `find ... -type f` finds it 
+FIND_EXCLUDE_PHRASE="$FIND_EXCLUDE_PHRASE -not -path './test/lib/alimonitor-services/cache/*'"
+
 FINDCMD="find . -type f $FIND_EXCLUDE_PHRASE"
 
 f() {
     for p in $(bash -c "$FINDCMD"); do
-        test $(tail -c1 "$p" | tr $'\n' '_') != '_' && echo "No newline at end of $p";
-        test "$(tail -c2 "$p" | tr $'\n' '_')" = '__' && echo "More than one newline at the end of $p";
+        if [ $(echo $p | grep "$(git ls-files -md; git diff --cached --name-only)") ]; then
+            test $(tail -c1 "$p" | tr $'\n' '_') != '_' && echo "No newline at end of $p";
+            test "$(tail -c2 "$p" | tr $'\n' '_')" = '__' && echo "More than one newline at the end of $p";
+        fi;
     done;
 }
 

test/lib/alimonitor-services/syncManager.test.js

@@ -0,0 +1,71 @@
+/**
+ * @license
+ * Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+ * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+ * All rights not expressly granted are reserved.
+ *
+ * This software is distributed under the terms of the GNU General Public
+ * License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+ *
+ * In applying this license CERN does not waive the privileges and immunities
+ * granted to it by virtue of its status as an Intergovernmental Organization
+ * or submit itself to any jurisdiction.
+ */
+
+const { rctData: { detectors } } = require('../../../app/lib/config/configProvider.js');
+const { syncManager } = require('../../../app/lib/alimonitor-services/SyncManager.js');
+const { databaseManager: { repositories: {
+    RunRepository,
+    RunDetectorsRepository,
+    DetectorSubsystemRepository,
+},
+} } = require('../../../app/lib/database/DatabaseManager.js');
+const { generateRandomBookkeepingCachedRawJsons, cleanCachedBkpData } = require('./testutil/cache-for-test.js');
+const assert = require('assert');
+
+module.exports = () => describe('SyncManager suite', () => {
+    before('should fetch detectors data from DB the same as in config', async () => await DetectorSubsystemRepository
+        .findAll({ raw: true })
+        .then((detectoSubsystemData) => detectoSubsystemData.map(({ name }) => name))
+        .then((detectoSubsystemNames) => assert.deepStrictEqual(detectoSubsystemNames.sort(), detectors.sort())));
+
+    describe('BookkeepingService suite', () => {
+        describe('with artificial cache data', () => {
+            before(() => {
+                generateRandomBookkeepingCachedRawJsons();
+            });
+
+            after(() => {
+                cleanCachedBkpData();
+            });
+
+            it('should performe sync with random data withour major errors', async () => {
+                assert.strictEqual(await syncManager.services.bookkeepingService.setSyncTask(), true);
+            });
+
+            it('should fetch some run data directly from DB', async () =>
+                await RunRepository
+                    .findAll({ raw: true })
+                    .then((data) => assert(data.length > 0)));
+
+            it('should fetch some run_detector data directly from DB', async () =>
+                await RunDetectorsRepository
+                    .findAll({ raw: true })
+                    .then((data) => assert(data.length > 0)));
+        });
+
+        describe('without artificial cache data', () => {
+            before(() => {
+                syncManager.services.bookkeepingService.forceToUseOnlyCache = true;
+            });
+
+            after(() => {
+                syncManager.services.bookkeepingService.forceToUseOnlyCache = false;
+            });
+
+            it('should performe sync with major error', async () => {
+                assert.strictEqual(await syncManager.services.bookkeepingService.setSyncTask(), false);
+            });
+        });
+    });
+});