[ORCT-118] monalisa service mc test

app/lib/alimonitor-services/BookkeepingService.js

@@ -136,7 +136,7 @@ class BookkeepingService extends AbstractServiceSynchronizer {
                 where: {
                     name: period.name,
                 },
-                default: {
+                defaults: {
                     name: period.name,
                     year: period.year,
                     BeamTypeId: beamType.id,

app/lib/alimonitor-services/MonalisaService.js

@@ -97,7 +97,7 @@ class MonalisaService extends AbstractServiceSynchronizer {
                 where: {
                     name: period.name,
                 },
-                default: {
+                defaults: {
                     name: period.name,
                     year: period.year,
                     BeamTypeId: beamType.id,

app/lib/alimonitor-services/MonalisaServiceMC.js

@@ -16,7 +16,6 @@
 const AbstractServiceSynchronizer = require('./AbstractServiceSynchronizer.js');
 const Utils = require('../utils');
 const { ServicesEndpointsFormatter, ServicesDataCommons: { extractPeriod } } = require('./helpers');
-
 const config = require('../config/configProvider.js');
 
 const { databaseManager: {
@@ -82,17 +81,18 @@ class MonalisaServiceMC extends AbstractServiceSynchronizer {
          */
 
         simulationPass.anchoredPasses = parseListLikeString(simulationPass.anchoredPasses);
-        simulationPass.anchoredPeriods = parseListLikeString(simulationPass.anchoredPeriods);
+        simulationPass.anchoredPeriods = parseListLikeString(simulationPass.anchoredPeriods).map((periodName) =>
+            extractPeriod(periodName, simulationPass.beam_type));
         simulationPass.runs = parseListLikeString(simulationPass.runs).map((s) => Number(s));
 
         return simulationPass;
     }
 
     isDataUnitValid(simulationPass) {
         simulationPass.anchoredPeriods = simulationPass.anchoredPeriods
-            .filter((periodName) => {
+            .filter(({ year: periodYear }) => {
                 try {
-                    return extractPeriod(periodName).year >= config.dataFromYearIncluding;
+                    return periodYear >= config.dataFromYearIncluding;
                 } catch (error) {
                     this.logger.error(error);
                     return false;
@@ -105,76 +105,78 @@ class MonalisaServiceMC extends AbstractServiceSynchronizer {
     }
 
     async executeDbAction(simulationPass) {
-        const { beam_type } = simulationPass;
-
-        return await SimulationPassRepository.T.upsert({
+        return await SimulationPassRepository.upsert({
             name: simulationPass.name,
             PWG: simulationPass.pwg,
             jiraId: simulationPass.jiraId,
             description: simulationPass.description,
             requestedEvents: simulationPass.requestedEvents,
             outputSize: simulationPass.outputSize,
         })
-            .then(async ([_simulationPass, _]) => {
-                // Check periods;
-                simulationPass.anchoredPeriods.map(async (periodName) => await PeriodRepository.T.findOrCreate({
+            .then(async ([simulationPassDBInstance, _]) => {
+                await Promise.all(simulationPass.anchoredPeriods.map(async (period) =>
+                    this.findOrCreatePeriod(period)
+                        .then(async ([period, _]) => {
+                            const periodAddPromise = simulationPassDBInstance.addPeriod(period.id, { ignoreDuplicates: true });
+                            const dataPassPipelinePromises = this.findOrCreateAndAddDataPasses(simulationPass, simulationPassDBInstance, period);
+                            const runsAddPipeline = this.findOrCreateAndAddRuns(simulationPass, simulationPassDBInstance, period);
+
+                            await Promise.all([periodAddPromise, dataPassPipelinePromises, runsAddPipeline]);
+                        })));
+            });
+    }
+
+    async findOrCreatePeriod({ name: periodName, year: periodYear, beamType }) {
+        return await sequelize.transaction(async () => PeriodRepository.findOrCreate({
+            where: {
+                name: periodName,
+            },
+            defaults: {
+                name: periodName,
+                year: periodYear,
+                BeamTypeId: !beamType ? undefined : (await BeamTypeRepository.findOrCreate({
+                    where: {
+                        name: beamType,
+                    },
+                }))[0]?.id,
+            },
+        }));
+    }
+
+    async findOrCreateAndAddDataPasses(simulationPass, simulationPassDBInstance, period) {
+        const promises = simulationPass.anchoredPasses
+            .map((passSuffix) => sequelize.transaction(
+                () => DataPassRepository.findOrCreate({
                     where: {
-                        name: periodName,
+                        name: `${period.name}_${passSuffix}`,
                     },
-                    default: beam_type ? {
-                        name: periodName,
-                        BeamTypeId: await BeamTypeRepository.T.findOrCreate({
-                            where: {
-                                name: simulationPass.beam_type,
-                            },
-                        })[0]?.id,
-                    } : undefined,
-                })
-                    .then(async ([period, _]) => {
-                        // Add anchored period
-                        const periodAddPromise = sequelize.transaction((_t) => _simulationPass.addPeriod(period.id));
-
-                        // Add anchored passes
-                        const dataPassPipelinePromises = simulationPass.anchoredPasses
-                            .map(async (passSuffix) => await DataPassRepository.T.findOrCreate({
-                                where: {
-                                    name: `${period.name}_${passSuffix}`,
-                                },
-                                default: {
-                                    name: `${period.name}_${passSuffix}`,
-                                    PeriodId: period.id,
-                                },
-                            }).then(async ([dataPass, _]) => await sequelize.transaction((_t) => _simulationPass.addDataPass(dataPass.id))));
-
-                        // Add runs
-                        const runsAddPipeline = simulationPass.runs.map(async (runNumber) => {
-                            const run = await RunRepository.T.findOne({ where: { runNumber: runNumber } });
-                            if (!run) {
-                                const insertWithoutPeriod = simulationPass.anchoredPeriods.length > 1;
-                                if (insertWithoutPeriod) {
-                                    this.logger.warn(
-                                        `Neither run {runNumber: ${runNumber}} is found, nor can infer its belonging to period, because multiple 
-                                        periods (${simulationPass.anchoredPeriods}) are anchored to simulation pass ${simulationPass.name}`,
-                                    );
-                                }
-
-                                await RunRepository.T.findOrCreate({
-                                    where: {
-                                        runNumber,
-                                    },
-                                    default: {
-                                        runNumber,
-                                        PeriodId: insertWithoutPeriod ? undefined : period.id,
-                                    },
-                                });
-                            }
-                            return await sequelize.transaction((_t) => _simulationPass.addRun(runNumber, { ignoreDuplicates: true }));
-                        });
-
-                        // Summary
-                        return await Promise.all([periodAddPromise, dataPassPipelinePromises, runsAddPipeline].flat());
-                    }));
+                    defaults: {
+                        name: `${period.name}_${passSuffix}`,
+                        PeriodId: period.id,
+                    },
+                }).then(([dataPass, _]) => simulationPassDBInstance.addDataPass(dataPass.id,
+                    { ignoreDuplicates: true })),
+            ));
+        return await Promise.all(promises);
+    }
+
+    async findOrCreateAndAddRuns(simulationPass, simulationPassDBInstance, period) {
+        const promises = simulationPass.runs.map((runNumber) => sequelize.transaction(async () => {
+            const insertWithoutPeriod = simulationPass.anchoredPeriods.length > 1;
+            await RunRepository.findOrCreate({
+                where: {
+                    runNumber,
+                },
+                defaults: {
+                    runNumber,
+                    PeriodId: insertWithoutPeriod ? undefined : period.id,
+                },
             });
+
+            return await simulationPassDBInstance.addRun(runNumber, { ignoreDuplicates: true });
+        }));
+
+        return await Promise.all(promises);
     }
 }
 

app/lib/alimonitor-services/helpers/ServicesDataCommons.js

@@ -54,7 +54,7 @@ function extractPeriodYear(name) {
  * @param {string|undefined} beamType type of beam, p-p, p-Pb, ...
  * @returns {Object} period {name, year, beamType}
  */
-function extractPeriod(name, beamType) {
+function extractPeriod(name, beamType = undefined) {
     const [extractedName] = name.split('_');
     if (! /LHC[0-9]{2}[a-z]+/.test(extractedName)) {
         throw new Error(`Incorrect period name ${extractedName} extracted from ${name}`);

test/lib/alimonitor-services/syncManager.test.js

@@ -17,6 +17,7 @@ const { syncManager: {
     services: {
         bookkeepingService,
         monalisaService,
+        monalisaServiceMC,
     },
 } } = require('../../../app/lib/alimonitor-services/SyncManager.js');
 const { databaseManager: {
@@ -25,16 +26,21 @@ const { databaseManager: {
         RunDetectorsRepository,
         DetectorSubsystemRepository,
         DataPassRepository,
+        SimulationPassRepository,
     },
     models: {
         Run,
         Period,
+        DataPass,
     },
 } } = require('../../../app/lib/database/DatabaseManager.js');
-const { generateRandomBookkeepingCachedRawJsons, cleanCachedBkpData } = require('./testutil/bookkeeping-cache-test-data.js');
-const { generateRandomMonalisaCachedRawJsons, cleanCachedMonalisaData } = require('./testutil/monalisa-cache-test-data.js');
+const { generateRandomBookkeepingCachedRawJsons } = require('./testutil/bookkeeping-cache-test-data.js');
+const { generateRandomMonalisaCachedRawJsons } = require('./testutil/monalisa-cache-test-data.js');
+const { generateRandomMonalisaMontecarloCachedRawJsons } = require('./testutil/monalisa-montecarlo-cache-test-data.js');
+
 const assert = require('assert');
 const { expect } = require('chai');
+const { truncateDatabase } = require('./testutil/db-utils.js');
 
 const artficialDataSizes = {
     bookkeepingService: {
@@ -46,10 +52,14 @@ const artficialDataSizes = {
         minDetailsPerOneDataPass: 1,
         maxDetailsPerOneDataPass: 10,
     },
+    monalisaServiceMC: {
+        simulationPassesNo: 10,
+    },
 };
 
 module.exports = () => describe('SyncManager suite', () => {
-    before(() => {
+    before(async () => {
+        await truncateDatabase();
         generateRandomBookkeepingCachedRawJsons(
             artficialDataSizes.bookkeepingService.runsInOneFile,
             artficialDataSizes.bookkeepingService.filesNo,
@@ -59,11 +69,9 @@ module.exports = () => describe('SyncManager suite', () => {
             artficialDataSizes.monalisaService.minDetailsPerOneDataPass,
             artficialDataSizes.monalisaService.maxDetailsPerOneDataPass,
         );
-    });
-
-    after(() => {
-        cleanCachedBkpData();
-        cleanCachedMonalisaData();
+        generateRandomMonalisaMontecarloCachedRawJsons(
+            artficialDataSizes.monalisaServiceMC.simulationPassesNo,
+        );
     });
 
     it('should fetch detectors data from DB the same as in config', async () => await DetectorSubsystemRepository
@@ -73,7 +81,7 @@ module.exports = () => describe('SyncManager suite', () => {
 
     describe('BookkeepingService suite', () => {
         describe('with artificial cache data', () => {
-            it('should performe sync with random data withour major errors', async () => {
+            it('should performe sync of runs with random data withour major errors', async () => {
                 bookkeepingService.useCacheJsonInsteadIfPresent = true;
                 expect(await bookkeepingService.setSyncTask()).to.be.equal(true);
             });
@@ -92,7 +100,7 @@ module.exports = () => describe('SyncManager suite', () => {
 
     describe('MonalisaService suite', () => {
         describe('with artificial cache data', () => {
-            it('should performe sync with random data without major errors', async () => {
+            it('should performe sync od data passes with random data without major errors', async () => {
                 monalisaService.useCacheJsonInsteadIfPresent = true;
                 assert.strictEqual(await monalisaService.setSyncTask(), true);
             });
@@ -106,4 +114,36 @@ module.exports = () => describe('SyncManager suite', () => {
             });
         });
     });
+
+    describe('MonalisaServiceMC suite', () => {
+        describe('with artificial cache data', () => {
+            it('should performe sync of simulation passes with random data without major errors', async () => {
+                monalisaServiceMC.useCacheJsonInsteadIfPresent = true;
+                assert.strictEqual(await monalisaServiceMC.setSyncTask(), true);
+            });
+
+            it('should fetch some simulation passes with associated Periods, Runs and DataPasses directly from DB', async () => {
+                const data = await SimulationPassRepository
+                    .findAll({ include: [Run, Period, DataPass] });
+
+                expect(data).to.length.greaterThan(0); //TODO
+
+                /**
+                 * Check if all simulation passes have associations with some periods and some data passes
+                 * Each simulation pass has to have association with at least one data pass and with at least one period,
+                 * The opposite means error in logic of the service;
+                 */
+                expect(data.map(({ Periods }) => Periods).filter((Periods) => Periods?.length > 0)).to.be.lengthOf(data.length);
+                expect(data.map(({ DataPasses }) => DataPasses).filter((DataPasses) => DataPasses?.length > 0)).to.be.lengthOf(data.length);
+
+                /**
+                 * Runs associated with one simulation pass,
+                 * should be noticed in database regardless
+                 * of their presence in the Bookkeeping
+                 * or lack of important features
+                 */
+                expect(data.map(({ Runs }) => Runs).filter((Runs) => Runs?.length > 0)).to.be.lengthOf(data.length);
+            });
+        });
+    });
 });

test/lib/alimonitor-services/testutil/bookkeeping-cache-test-data.js

@@ -3,11 +3,11 @@ const path = require('path');
 const { Cacher } = require('../../../../app/lib/alimonitor-services/helpers');
 const { rctData: { detectors } } = require('../../../../app/lib/config/configProvider.js');
 
-const { randint, choice, universalNoncontextualArrayDataGenerator, randomPeriodName } = require('./common.js');
+const { randint, choice, universalNoncontextualArrayDataGenerator, randomPeriodName, randomRunNumber, randomBeamType } = require('./common.js');
 
 const dataUnitDefinition = {
-    runNumber: () => randint(1000000, 9000000),
-    lhcPeriod: () => randomPeriodName(),
+    runNumber: randomRunNumber,
+    lhcPeriod: randomPeriodName,
     timeO2Start: () => randint(100000000, 200000000),
     timeO2End: () => randint(100000000, 200000000),
     timeTrgStart: () => randint(200000000, 300000000),
@@ -20,7 +20,7 @@ const dataUnitDefinition = {
     aliceDipoleCurrent: () => randint(10000, 20000),
     aliceDipolePolarity: () => choice(['POSITIVE', 'NEGATIVE']),
     fillNumber: () => randint(5000, 9000),
-    pdpBeamType: () => choice(['pp', 'PbPb', 'pPb']),
+    pdpBeamType: randomBeamType,
 };
 
 const getBkpSourceFiles = (offset, limit) =>

test/lib/alimonitor-services/testutil/common.js

@@ -11,10 +11,19 @@
  * granted to it by virtue of its status as an Intergovernmental Organization
  * or submit itself to any jurisdiction.
  */
+const { rctData: {
+    mapping: {
+        beamType: {
+            values: beamTypes,
+        },
+    },
+} } = require('../../../../app/lib/config/configProvider.js');
 
 const randint = (min = 0, max = 0) => Math.round(Math.random() * (max - min) + min);
 const choice = (arr) => arr[Math.floor(Math.random() * arr.length)];
 const randomPeriodName = () => `LHC${choice([22, 18])}${choice('abceadbfarebivaavgauvgzxvcm')}`;
+const randomRunNumber = () => randint(1000000, 9000000);
+const randomBeamType = () => choice(Object.values(beamTypes));
 
 /**
  * Generate random data with structure defined in given argument
@@ -67,6 +76,8 @@ module.exports = {
     randint,
     choice,
     randomPeriodName,
+    randomRunNumber,
+    randomBeamType,
     universalNoncontextualDataUnitGenerator,
     universalNoncontextualArrayDataGenerator,
 };

test/lib/alimonitor-services/testutil/db-utils.js

@@ -0,0 +1,26 @@
+/**
+ * @license
+ * Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+ * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+ * All rights not expressly granted are reserved.
+ *
+ * This software is distributed under the terms of the GNU General Public
+ * License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+ *
+ * In applying this license CERN does not waive the privileges and immunities
+ * granted to it by virtue of its status as an Intergovernmental Organization
+ * or submit itself to any jurisdiction.
+ */
+
+const { databaseManager: {
+    sequelize,
+} } = require('../../../../app/lib/database/DatabaseManager.js');
+
+const truncateDatabase = async () => {
+    await sequelize.getQueryInterface().bulkDelete('periods', {}, { truncate: true, cascade: true });
+    await sequelize.getQueryInterface().bulkDelete('simulation_passes', {}, { truncate: true, cascade: true });
+};
+
+module.exports = {
+    truncateDatabase,
+};