Environment usage refactor

.github/workflows/run-tests.yml

@@ -1,5 +1,5 @@
 name: Tests
-
+## TODO
 on:
   pull_request:
   push:

.vscode/settings.json

@@ -11,7 +11,6 @@
         ".eslintrc": true,
         ".nycrc": true,
         ".github": true,
-        ".env-default": true,
         "codecov.yml": true,
 
         "LICENSE": true,

app/application.js

@@ -206,7 +206,9 @@ class RunConditionTableApplication {
         }
 
         await this.connectServices();
-        this.setSyncAllTask();
+        if (config.syncTaskAtStart) {
+            this.setSyncAllTask();
+        }
         this.logger.info('RCT app started');
     }
 

app/config/index.js

@@ -22,10 +22,12 @@ module.exports = Object.freeze({
     // App config
     winston: ResProvider.winston(),
     database: ResProvider.database(),
-    syncTaskByDefault: process.env['RCT_SYNC_TASK_BY_DEFAULT'] ? process.env['RCT_SYNC_TASK_BY_DEFAULT'] : true,
+    syncTaskAtStart: process.env['RCT_SYNC_TASK_AT_START']?.trim().toLowerCase() === 'true' ? process.env['RCT_SYNC_TASK_AT_START'] : false,
     databasePersistance: require('./databasePersistance.js'),
     public: require('./public.js'),
 
+    // External services config
+    services: require('./services.js'),
     // RCT data config
     dataFromYearIncluding: 2018,
 
@@ -36,5 +38,5 @@ module.exports = Object.freeze({
         stack: (logger, er) => logger.error(er.stack),
         object: (logger, er) => logger.error(JSON.stringify(er, null, 2)),
     },
-    defaultErrorsLogginDepth: 'object',
+    defaultErrorsLogginDepth: process.env['RCT_ERR_DEPTH']?.length() > 0 ? process.env['RCT_ERR_DEPTH'] : 'object',
 });

app/config/services.js

@@ -0,0 +1,52 @@
+/**
+ *
+ * @license
+ * Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+ * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+ * All rights not expressly granted are reserved.
+ *
+ * This software is distributed under the terms of the GNU General Public
+ * License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+ *
+ * In applying this license CERN does not waive the privileges and immunities
+ * granted to it by virtue of its status as an Intergovernmental Organization
+ * or submit itself to any jurisdiction.
+ */
+const ResProvider = require('../lib/ResProvider.js');
+
+const services = {
+    bookkeeping: {
+        url: {
+            rct: 'http://rct-bookkeeping.cern.ch:4000/api/runs',
+            ali: ResProvider.endpointFor('BK_RUNS'),
+        },
+    },
+    monalisa: {
+        url: {
+            dataPassesRaw: ResProvider.endpointFor('ML_DP_RAW'),
+            dataPassesDetailed: ResProvider.endpointFor('ML_DP_DET'),
+
+            mcRaw: ResProvider.endpointFor('ML_MC_RAW'),
+            mcDetailed: ResProvider.endpointFor('ML_MC_DET'),
+            mcDetTag: ResProvider.endpointFor('ML_MC_TAG'),
+        },
+    },
+};
+
+/*
+ * Endpoints bases
+ * ali: 'https://ali-bookkeeping.cern.ch/api/runs?filter[definitions]=PHYSICS',
+ * dataPassesRaw: 'https://alimonitor.cern.ch/production/raw.jsp?res_path=json',
+ * dataPassesDetailed: 'https://alimonitor.cern.ch/raw/raw_details.jsp?timesel=0&res_path=json',
+ * mcRaw: 'https://alimonitor.cern.ch/MC/?res_path=json',
+ * mcDetailed: 'https://alimonitor.cern.ch/job_events.jsp?timesel=0&res_path=json',
+ * mcDetTag: 'https://alimonitor.cern.ch/MC/prodDetails.jsp?res_path=json',
+ */
+
+// LHC21i3f3
+// eslint-disable-next-line max-len
+//E rawDataDetalied: 'https://alimonitor.cern.ch/production/raw_details.jsp?timesel=0&filter_jobtype=OCT+-+async+production+for+pilot+beam+pass+3%2C+O2-2763&res_path=json',
+// eslint-disable-next-line max-len
+//E mcRawDataDetailed: 'https://alimonitor.cern.ch/job_events.jsp?timesel=0&owner=aliprod&filter_jobtype=Pb-Pb%2C+5.02+TeV+-+HIJING+%2B+nuclei+Geant4+with+modified+material+budget+%2B4.5%+(Pb-Pb+Pass3)%2C+50-90%+centrality%2C+ALIROOT-8784&res_path=json',
+
+module.exports = services;

app/lib/ResProvider.js

@@ -52,7 +52,7 @@ class ResProvider {
                 return Utils.switchCase(onFailureAction, {
                     error: ResProvider.onFailureAction_error,
                     warn: ResProvider.onFailureAction_warn,
-                    no: () => null,
+                    no: () => res,
                 })(res, objDefinition);
             }
         }
@@ -85,6 +85,7 @@ class ResProvider {
     static onFailureAction_warn(res, objDef) {
         const mess = ResProvider.nulledMessageGetter(res, objDef);
         logger.warn(mess);
+        return res;
     }
 
     static securityFilesContentProvider(fileNames, description, envVarName, supressLogs = false) {
@@ -215,6 +216,26 @@ class ResProvider {
         }
         return process.env.ALIMONITOR_PASSPHRASE;
     }
+
+    static getServiceEndpoint(serviceAbbr) {
+        // BK-RUNS, ML-DP, ML-MC
+        const varsDef = {};
+        varsDef[`RCT_EP_${serviceAbbr}_PROT`] = 'prot';
+        varsDef[`RCT_EP_${serviceAbbr}_HOST`] = 'host';
+        varsDef[`RCT_EP_${serviceAbbr}_PORT`] = 'port';
+        varsDef[`RCT_EP_${serviceAbbr}_PATH`] = 'path';
+        const p = ResProvider.viaEnvVars(varsDef, null, 'warn');
+        const { host } = p;
+        let { path } = p;
+        // eslint-disable-next-line prefer-destructuring
+        let prot = p['prot'];
+        prot = prot ? prot : 'https';
+        // eslint-disable-next-line prefer-destructuring
+        let port = p['port'];
+        port = port ? `:${port}` : '';
+        path = path ? path.trim().replace(/^\/*/, '') : '';
+        return `${prot}://${host}${port}/${path}`;
+    }
 }
 
 logger = new Log(ResProvider.name);

app/lib/alimonitor-services/AbstractServiceSynchronizer.js

@@ -192,9 +192,9 @@ class AbstractServiceSynchronizer {
             this.monitor.logResults();
         } catch (fatalError) {
             this.logger.error(fatalError.stack);
-            if (/ECONREFUSED|ENOTFOUND|ECONNRESET|ETIMEDOUT|ECONNABORTED|EHOSTUNREACH|EAI_AGAIN/.test(fatalError.name + fatalError.message)) {
+            if (/ECONNREFUSED|ENOTFOUND|ECONNRESET|ETIMEDOUT|ECONNABORTED|EHOSTUNREACH|EAI_AGAIN/.test(fatalError.name + fatalError.message)) {
                 this.forceStop = true;
-                this.logger.error(`terminated due to fatal error ${fatalError.name}`);
+                this.logger.error(`terminated due to fatal error ${fatalError.name} for endpoint: ${endpoint}`);
             }
         }
     }
@@ -262,7 +262,7 @@ class AbstractServiceSynchronizer {
     }
 
     isConnected() {
-        return this.dbClient._connected;
+        return this.dbclient?._connected;
     }
 
     async restart() {

app/lib/alimonitor-services/ServicesEndpointsFormatter.js

@@ -13,24 +13,7 @@
  * or submit itself to any jurisdiction.
  */
 
-const services = {
-    bookkeeping: {
-        url: {
-            rct: 'http://rct-bookkeeping.cern.ch:4000/api/runs',
-            ali: 'https://ali-bookkeeping.cern.ch/api/runs?filter[definitions]=PHYSICS',
-        },
-    },
-    monalisa: {
-        url: {
-            dataPassesRaw: 'https://alimonitor.cern.ch/production/raw.jsp?res_path=json',
-            dataPassesDetailed: 'https://alimonitor.cern.ch/raw/raw_details.jsp?timesel=0&res_path=json',
-
-            mcRaw: 'https://alimonitor.cern.ch/MC/?res_path=json',
-            mcDetailed: 'https://alimonitor.cern.ch/job_events.jsp?timesel=0&res_path=json',
-            mcDetTag: 'https://alimonitor.cern.ch/MC/prodDetails.jsp?res_path=json',
-        },
-    },
-};
+const { services } = require('../config/configProvider.js');
 
 class ServicesEnpointsFormatter {
     static bookkeeping(page, limit) {
@@ -55,8 +38,3 @@ class ServicesEnpointsFormatter {
 }
 
 module.exports = ServicesEnpointsFormatter;
-// LHC21i3f3
-// eslint-disable-next-line max-len
-//E rawDataDetalied: 'https://alimonitor.cern.ch/production/raw_details.jsp?timesel=0&filter_jobtype=OCT+-+async+production+for+pilot+beam+pass+3%2C+O2-2763&res_path=json',
-// eslint-disable-next-line max-len
-//E mcRawDataDetailed: 'https://alimonitor.cern.ch/job_events.jsp?timesel=0&owner=aliprod&filter_jobtype=Pb-Pb%2C+5.02+TeV+-+HIJING+%2B+nuclei+Geant4+with+modified+material+budget+%2B4.5%+(Pb-Pb+Pass3)%2C+50-90%+centrality%2C+ALIROOT-8784&res_path=json',

docker/Dockerfile.application → docker/application.dockerfile

@@ -1,23 +1,20 @@
-FROM node:16.9.1-alpine3.12 as base
+# TODO upgrade node version
+FROM node:16.20.0-buster as base 
 
     WORKDIR /opt/RunConditionTable
-
-    RUN apk add --no-cache \
-        bash=5.0.17-r0
-
-
-FROM base as devdependencies
-
+    RUN apt update -y && apt install -y \
+        netcat \
+        bash
     COPY ./package*.json ./
     RUN npm --silent ci
 
 
-FROM devdependencies as development
+FROM base as development
 
     CMD [ "./scripts/check-host-and-exec.sh", "o2-rct_database", "5432", "10", "--", "npm", "run", "start:dev" ]
 
 
-FROM devdependencies as test
+FROM base as test
 
     COPY ./.eslintrc ./
     COPY ./.nycrc ./
@@ -26,4 +23,4 @@ FROM devdependencies as test
     COPY ./app ./app
     COPY ./scripts ./scripts
 
-    CMD [ "./scripts/check-host-and-exec.sh", "o2-rct_database", "5432", "10", "--", "npm", "run", "test" ]
+    CMD [ "./scripts/check-host-and-exec.sh", "o2-rct_database-test", "5432", "10", "--", "npm", "run", "test" ]

docker/database.dockerfile

@@ -0,0 +1,18 @@
+FROM postgres:14.2-bullseye as base
+
+WORKDIR /postgres/run
+
+FROM base as development
+    RUN apt update -y && \
+        apt install -y \
+        sudo \
+        procps \
+        inotify-tools
+
+FROM base as test
+
+
+
+
+
+

docker/dev.env

@@ -0,0 +1,96 @@
+ENV_MODE=dev
+RUNNING_ENV=DOCKER
+
+## loggin
+RCT_LOG_FILENAME=${RCT_LOG_FILENAME:-reports/rctlogs.txt}
+RCT_LOG_FILE_LOGLEV=${RCT_LOG_FILE_LOGLEV:-warn}
+RCT_LOG_CONSOLE_LOGLEV=${RCT_LOG_CONSOLE_LOGLEV:-debug}
+RCT_LOG_CONSOLE_SYSD=${RCT_LOG_CONSOLE_SYSD:-}
+
+## http server
+RCT_HTTP_PORT=${RCT_HTTP_PORT:-8081}
+RCT_HOSTNAME=${RCT_HOSTNAME:-localhost}
+RCT_TLS_ENABLED=${RCT_TLS_ENABLED:-false}
+
+RCT_JWT_SECRET=${RCT_JWT_SECRET:-secret}
+RCT_JWT_EXPIRATION=${RCT_JWT_EXPIRATION:-2h}
+
+## external services connections
+RCT_SYNC_TASK_AT_START=${RCT_SYNC_TASK_AT_START:-false}
+
+RCT_CERT_PATH=${RCT_CERT_PATH:-}
+ALIMONITOR_PASSPHRASE=${ALIMONITOR_PASSPHRASE:-}
+CERN_SOCKS=${CERN_SOCKS:-true}
+
+### endpoints
+_DEF_BK_HOST=ali-bookkeeping.cern.ch
+_DEF_ML_HOST=alimonitor.cern.ch
+_DEF_PORT=443
+_DEF_ML_PROT=https
+
+### bkp
+RCT_EP_BK_RUNS_PROT=https
+RCT_EP_BK_RUNS_HOST=${RCT_EP_BK_RUNS_HOST:-$_DEF_BK_HOST}
+RCT_EP_BK_RUNS_PORT=${RCT_EP_BK_RUNS_PORT:-$_DEF_PORT}
+RCT_EP_BK_RUNS_PATH=${RCT_EP_BK_RUNS_PATH:-/api/runs?filter[definitions]=PHYSICS}
+
+
+### data passes
+RCT_EP_ML_DP_RAW_PROT=$_DEF_ML_PROT
+RCT_EP_ML_DP_RAW_HOST=${RCT_EP_ML_DP_RAW_HOST:-$_DEF_ML_HOST}
+RCT_EP_ML_DP_RAW_PORT=${RCT_EP_ML_DP_RAW_PORT:-$_DEF_PORT}
+RCT_EP_ML_DP_RAW_PATH=${RCT_EP_ML_DP_RAW_PATH:-/production/raw.jsp?res_path=json}
+
+
+RCT_EP_ML_DP_DET_PROT=$_DEF_ML_PROT
+RCT_EP_ML_DP_DET_HOST=${RCT_EP_ML_DP_DET_HOST:-$_DEF_ML_HOST}
+RCT_EP_ML_DP_DET_PORT=${RCT_EP_ML_DP_DET_PORT:-$_DEF_PORT}
+RCT_EP_ML_DP_DET_PATH=${RCT_EP_ML_DP_DET_PATH:-/raw/raw_details.jsp?timesel=0&res_path=json}
+
+
+### simulation passes
+RCT_EP_ML_MC_RAW_PROT=$_DEF_ML_PROT
+RCT_EP_ML_MC_RAW_HOST=${RCT_EP_ML_MC_RAW_HOST:-$_DEF_ML_HOST}
+RCT_EP_ML_MC_RAW_PORT=${RCT_EP_ML_MC_RAW_PORT:-$_DEF_PORT}
+RCT_EP_ML_MC_RAW_PATH=${RCT_EP_ML_MC_RAW_PATH:-/MC/?res_path=json}
+
+RCT_EP_ML_MC_DET_PROT=$_DEF_ML_PROT
+RCT_EP_ML_MC_DET_HOST=${RCT_EP_ML_MC_DET_HOST:-$_DEF_ML_HOST}
+RCT_EP_ML_MC_DET_PORT=${RCT_EP_ML_MC_DET_PORT:-$_DEF_PORT}
+RCT_EP_ML_MC_DET_PATH=${RCT_EP_ML_MC_DET_PATH:-/job_events.jsp?timesel=0&res_path=json}
+
+RCT_EP_ML_MC_TAG_PROT=$_DEF_ML_PROT
+RCT_EP_ML_MC_TAG_HOST=${RCT_EP_ML_MC_TAG_HOST:-$_DEF_ML_HOST}
+RCT_EP_ML_MC_TAG_PORT=${RCT_EP_ML_MC_TAG_PORT:-$_DEF_PORT}
+RCT_EP_ML_MC_TAG_PATH=${RCT_EP_ML_MC_TAG_PATH:-/MC/prodDetails.jsp?res_path=json}
+
+
+## openid
+RCT_OPENID_ID=${RCT_OPENID_ID:-}
+RCT_OPENID_SECRET=${RCT_OPENID_SECRET:-}
+RCT_OPENID_REDIRECT=${RCT_OPENID_REDIRECT:-}
+RCT_OPENID_WELL_KNOWN=${RCT_OPENID_WELL_KNOWN:-}
+
+
+## database configuration
+### common vars
+RCT_DB_HOST=${RCT_DB_HOST:-o2-rct_database}
+RCT_DB_UNIXSS=${RCT_DB_UNIXSS:-/var/run/postgresql}
+RCT_DB_NAME=${RCT_DB_NAME:-rct-db}
+RCT_DB_PORT=5432
+RCT_DB_USERNAME=${RCT_DB_USERNAME:-rct-user}
+RCT_DB_PASSWORD=${RCT_DB_PASSWORD:-rct-passwd}
+
+### docker contatiner conf
+POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
+POSTGRES_USER=${POSTGRES_USER:-postgres}
+POSTGRES_DB=${POSTGRES_DB:-postgres}
+PGDATA=${PGDATA:-/var/lib/postgresql/data/pgdata}
+POSTGRES_HOST_AUTH_METHOD=scram-sha-256
+
+MOCK_DB=${MOCK_DB:-false}
+OTHER_SQL_MODIFY_DAEMON=${OTHER_SQL_MODIFY_DAEMON:-true}
+MAIN_SQL_MODIFY_DAEMON=${MAIN_SQL_MODIFY_DAEMON:-true}
+
+
+