[ORCT-118] - monalisa mc sync to sequelize

app/lib/alimonitor-services/BookkeepingService.js

@@ -15,7 +15,7 @@
 
 const AbstractServiceSynchronizer = require('./AbstractServiceSynchronizer.js');
 const Utils = require('../utils');
-const ServicesDataCommons = require('./ServicesDataCommons.js');
+const { extractPeriod, mapBeamTypeToCommonFormat } = require('./ServicesDataCommons.js');
 const EndpintFormatter = require('./ServicesEndpointsFormatter.js');
 const { databaseManager: {
     repositories: {
@@ -94,7 +94,7 @@ class BookkeepingService extends AbstractServiceSynchronizer {
 
             this.coilsCurrentsFieldsParsing(run, 'l3_current_val', 'l3_current_polarity', 'l3CurrentVal');
             this.coilsCurrentsFieldsParsing(run, 'dipole_current_val', 'dipole_current_polarity', 'dipoleCurrentVal');
-            ServicesDataCommons.mapBeamTypeToCommonFormat(run);
+            mapBeamTypeToCommonFormat(run);
             run.fillNumber = Number(run.fillNumber);
             return run;
         } catch (e) {
@@ -126,7 +126,7 @@ class BookkeepingService extends AbstractServiceSynchronizer {
         delete run.detectorQualities;
         delete run.beamType;
 
-        const year = ServicesDataCommons.extractPeriodYear(periodName);
+        const period = extractPeriod(periodName, beamType);
         const { detectorsNameToId } = this;
 
         return await BeamTypeRepository.T.findOrCreate({
@@ -136,8 +136,8 @@ class BookkeepingService extends AbstractServiceSynchronizer {
         })
             .then(async ([beamType, _]) => await PeriodRepository.T.findOrCreate({
                 where: {
-                    name: periodName,
-                    year,
+                    name: period.name,
+                    year: period.year,
                     BeamTypeId: beamType.id,
                 },
             }))
@@ -147,8 +147,8 @@ class BookkeepingService extends AbstractServiceSynchronizer {
                         error: e.message,
                         meta: {
                             explicitValues: {
-                                name: periodName,
-                                year,
+                                name: period.name,
+                                year: period.year,
                                 BeamTypeId: beamType.id,
                             },
                             implicitValues: {

app/lib/alimonitor-services/MonalisaService.js

@@ -15,7 +15,7 @@
 
 const AbstractServiceSynchronizer = require('./AbstractServiceSynchronizer.js');
 const Utils = require('../utils');
-const ServicesDataCommons = require('./ServicesDataCommons.js');
+const { mapBeamTypeToCommonFormat, extractPeriod } = require('./ServicesDataCommons.js');
 const EndpointsFormatter = require('./ServicesEndpointsFormatter.js');
 const MonalisaServiceDetails = require('./MonalisaServiceDetails.js');
 const config = require('../config/configProvider.js');
@@ -84,7 +84,7 @@ class MonalisaService extends AbstractServiceSynchronizer {
     dataAdjuster(dp) {
         dp = Utils.filterObject(dp, this.ketpFields);
         dp.outputSize = dp.outputSize ? Number(dp.outputSize) : null;
-        dp.period = ServicesDataCommons.mapBeamTypeToCommonFormat(this.extractPeriod(dp));
+        dp.period = mapBeamTypeToCommonFormat(extractPeriod(dp.name, dp.beam_type));
         return dp;
     }
 
@@ -99,6 +99,9 @@ class MonalisaService extends AbstractServiceSynchronizer {
             .then(async ([beamType, _]) => await PeriodRepository.T.findOrCreate({
                 where: {
                     name: period.name,
+                },
+                default: {
+                    name: period.name,
                     year: period.year,
                     BeamTypeId: beamType.id,
                 },
@@ -122,26 +125,6 @@ class MonalisaService extends AbstractServiceSynchronizer {
             }))
             .then(async ([dataPass, _]) => await this.monalisaServiceDetails.setSyncTask({ dataUnit: dataPass }));
     }
-
-    extractPeriod(rowData) {
-        try {
-            const productionPrefix = rowData.name.slice(0, 6);
-            const period = {};
-            period.name = productionPrefix;
-            let year = parseInt(productionPrefix.slice(3, 5), 10);
-            if (year > 50) {
-                year += 1900;
-            } else {
-                year += 2000;
-            }
-            period.year = year;
-            period.beamType = rowData.beam_type;
-
-            return period;
-        } catch (e) {
-            return null;
-        }
-    }
 }
 
 module.exports = new MonalisaService();

app/lib/alimonitor-services/MonalisaServiceDetails.js

@@ -44,17 +44,28 @@ class MonalisaServiceDetails extends AbstractServiceSynchronizer {
             () => true,
             async (dbClient, v) => {
                 v.parentDataUnit = dataPass;
-                return await PeriodRepository.T.findOrCreate({
-                    where: {
-                        name: v.period,
-                    },
-                })
+
+                return (async () => {
+                    if (/LHC[0-9]{2}[a-z]+/.test(v.period)) {
+                        return await PeriodRepository.T.findOrCreate({
+                            where: {
+                                name: v.period,
+                            },
+                        });
+                    } else {
+                        this.logger.warn(`Incorrect period from monalisa ${v.period} for run ${v.runNumber} in data pass ${dataPass.name}`);
+                        return [undefined, undefined];
+                    }
+                })()
                     .then(async ([period, _]) => {
-                        v.PeriodId = period.id;
+                        v.PeriodId = period?.id;
                         return await RunRepository.T.findOrCreate({
                             where: {
                                 runNumber: v.runNumber,
-                                PeriodId: period.id,
+                            },
+                            defualt: {
+                                runNumber: v.runNumber,
+                                PeriodId: v.PeriodId,
                             },
                         });
                     })
@@ -77,7 +88,7 @@ class MonalisaServiceDetails extends AbstractServiceSynchronizer {
                         });
                     })
                     // eslint-disable-next-line no-unused-vars
-                    .then(async ([run, _]) => await sequelize.transaction((t) => run.addDataPasses(dataPass.id)));
+                    .then(async ([run, _]) => await sequelize.transaction((t) => run.addDataPasses(dataPass.id, { ignoreDuplicates: true })));
             },
         );
     }

app/lib/alimonitor-services/MonalisaServiceMC.js

@@ -16,9 +16,20 @@
 const AbstractServiceSynchronizer = require('./AbstractServiceSynchronizer.js');
 const Utils = require('../utils');
 const EndpointsFormatter = require('./ServicesEndpointsFormatter.js');
-const MonalisaServiceMCDetails = require('./MonalisaServiceMCDetails.js');
 const config = require('../config/configProvider.js');
 
+const { databaseManager: {
+    repositories: {
+        BeamTypeRepository,
+        PeriodRepository,
+        SimulationPassRepository,
+        DataPassRepository,
+        RunRepository,
+    },
+    sequelize,
+} } = require('../database/DatabaseManager.js');
+const { extractPeriod } = require('./ServicesDataCommons.js');
+
 class MonalisaServiceMC extends AbstractServiceSynchronizer {
     constructor() {
         super();
@@ -29,27 +40,34 @@ class MonalisaServiceMC extends AbstractServiceSynchronizer {
             name: 'name',
             runList: 'runs',
             generator: 'description',
-            jiraID: 'jira',
+            jiraID: 'jiraId',
             PWG: 'pwg',
-            requested_events: 'number_of_events',
+            requested_events: 'requestedEvents',
             collision_system: 'beam_type',
-            output_size: 'size',
-            anchor_production: 'anchor_productions',
-            anchor_pass: 'anchor_passes',
+            output_size: 'outputSize',
+            anchor_production: 'anchoredPeriods',
+            anchor_pass: 'anchoredPasses',
         };
-
-        this.monalisaServiceMCDetails = new MonalisaServiceMCDetails();
     }
 
     sync() {
         return this.syncPerEndpoint(
             EndpointsFormatter.mcRaw(),
             this.responsePreprocess.bind(this),
             this.dataAdjuster.bind(this),
-            (simulation_pass) => {
-                const { anchor_productions, anchor_passes } = simulation_pass;
-                return simulation_pass.period.year >= config.dataFromYearIncluding
-                    && anchor_productions.length != 0 && anchor_passes.length != 0;
+            (simulationPass) => {
+                simulationPass.anchoredPeriods = simulationPass.anchoredPeriods
+                    .filter((periodName) => {
+                        try {
+                            return extractPeriod(periodName).year >= config.dataFromYearIncluding;
+                        } catch (error) {
+                            this.logger.error(error);
+                            return false;
+                        }
+                    });
+
+                const { anchoredPeriods, anchoredPasses } = simulationPass;
+                return anchoredPeriods.length != 0 && anchoredPasses.length != 0;
                 // MC not anchored to any production or pass so drop out
             },
             this.dbAction.bind(this),
@@ -67,7 +85,8 @@ class MonalisaServiceMC extends AbstractServiceSynchronizer {
 
     dataAdjuster(sp) {
         sp = Utils.filterObject(sp, this.ketpFields);
-        sp.size = Number(sp.size);
+        sp.outputSize = Number(sp.outputSize);
+        sp.requestedEvents = Number(sp.requestedEvents);
 
         const parseListLikeString = (rawString) => Utils
             .replaceAll(rawString, /,|'|;"/, ' ')
@@ -80,56 +99,81 @@ class MonalisaServiceMC extends AbstractServiceSynchronizer {
          * there are extra commas at the begining of some samples
          */
 
-        sp.period = this.extractPeriod(sp);
-        sp.anchor_passes = parseListLikeString(sp.anchor_passes);
-        sp.anchor_productions = parseListLikeString(sp.anchor_productions);
+        sp.anchoredPasses = parseListLikeString(sp.anchoredPasses);
+        sp.anchoredPeriods = parseListLikeString(sp.anchoredPeriods);
         sp.runs = parseListLikeString(sp.runs).map((s) => Number(s));
 
         return sp;
     }
 
-    async dbAction(dbClient, d) {
-        d = Utils.adjusetObjValuesToSql(d);
-        const { period } = d;
-        const period_insert =
-            d?.period?.name ? `call insert_period(${period.name}, ${period.year}, ${period.beam_type});` : '';
-
-        const anchord_prod_sql = `${d.anchor_productions}::varchar[]`;
-        const anchord_passes_sql = `${d.anchor_passes}::varchar[]`;
-
-        const pgCommand = `${period_insert}; call insert_mc(
-            ${d.name}, 
-            ${d.description},
-            ${d.pwg},
-            ${anchord_prod_sql},
-            ${anchord_passes_sql},
-            ${d.jira},
-            ${d.number_of_events},
-            ${d.size}
-        ); call insert_mc_details(${d.name}, ${d.runs}::integer[], ${period.name});`;
-        return await dbClient.query(pgCommand);
-        // eslint-disable-next-line capitalized-comments
-        // return await Promise.all([dbClient.query(pgCommand), this.monalisaServiceMCDetails.sync(d)]);
-    }
-
-    extractPeriod(rowData) {
-        try {
-            const productionPrefix = rowData.name.slice(0, 6);
-            const period = {};
-            period.name = productionPrefix;
-            let year = parseInt(productionPrefix.slice(3, 5), 10);
-            if (year > 50) {
-                year += 1900;
-            } else {
-                year += 2000;
-            }
-            period.year = year;
-            period.beam_type = rowData.beam_type;
-
-            return period;
-        } catch (e) {
-            return null;
-        }
+    async dbAction(dbClient, simulationPass) {
+        const { beam_type } = simulationPass;
+
+        return await SimulationPassRepository.T.upsert({
+            name: simulationPass.name,
+            PWG: simulationPass.pwg,
+            jiraId: simulationPass.jiraId,
+            description: simulationPass.description,
+            requestedEvents: simulationPass.requestedEvents,
+            outputSize: simulationPass.outputSize,
+        })
+            .then(async ([_simulationPass, _]) => {
+                // Check periods;
+                simulationPass.anchoredPeriods.map(async (periodName) => await PeriodRepository.T.findOrCreate({
+                    where: {
+                        name: periodName,
+                    },
+                    default: beam_type ? {
+                        name: periodName,
+                        BeamTypeId: await BeamTypeRepository.T.findOrCreate({
+                            where: {
+                                name: simulationPass.beam_type,
+                            },
+                        })[0]?.id,
+                    } : undefined,
+                })
+                    .then(async ([period, _]) => {
+                        // Add anchored period
+                        const periodAddPromise = sequelize.transaction((_t) => _simulationPass.addPeriod(period.id));
+
+                        // Add anchored passes
+                        const dataPassPipelinePromises = simulationPass.anchoredPasses
+                            .map(async (passSuffix) => await DataPassRepository.T.findOrCreate({
+                                where: {
+                                    name: `${period.name}_${passSuffix}`,
+                                },
+                                default: {
+                                    name: `${period.name}_${passSuffix}`,
+                                    PeriodId: period.id,
+                                },
+                            }).then(async ([dataPass, _]) => await sequelize.transaction((_t) => _simulationPass.addDataPass(dataPass.id))));
+
+                        // Add runs
+                        const runsAddPipeline = simulationPass.runs.map(async (runNumber) => {
+                            const run = await RunRepository.T.findOne({ where: { runNumber: runNumber } });
+                            if (!run) {
+                                const insertWithoutPeriod = simulationPass.anchoredPeriods.length > 1;
+                                if (insertWithoutPeriod) {
+                                    this.logger.warn(
+                                        `Neither run {runNumber: ${runNumber}} is found, nor can infer its belonging to period, because multiple 
+                                        periods (${simulationPass.anchoredPeriods}) are anchored to simulation pass ${simulationPass.name}`,
+                                    );
+                                }
+
+                                await RunRepository.T.findOrCreate({
+                                    where: {
+                                        runNumber,
+                                        PeriodId: insertWithoutPeriod ? undefined : period.id,
+                                    },
+                                });
+                            }
+                            return await sequelize.transaction((_t) => _simulationPass.addRun(runNumber, { ignoreDuplicates: true }));
+                        });
+
+                        // Summary
+                        return await Promise.all([periodAddPromise, dataPassPipelinePromises, runsAddPipeline].flat());
+                    }));
+            });
     }
 }