[ORCT0-134] - filtering for new api

app/lib/domain/dtos/index.js

@@ -11,8 +11,8 @@
  * or submit itself to any jurisdiction.
  */
 
-const STDEntityDTO = require('./STDEntity.dto');
+const stdRequestDTO = require('./stdRequest.dto');
 
 module.exports = {
-    STDEntityDTO,
+    ...stdRequestDTO,
 };

app/lib/domain/dtos/stdRequest.dto.js

@@ -0,0 +1,37 @@
+/**
+ * @license
+ * Copyright CERN and copyright holders of ALICE O2. This software is
+ * distributed under the terms of the GNU General Public License v3 (GPL
+ * Version 3), copied verbatim in the file "COPYING".
+ *
+ * See http://alice-o2.web.cern.ch/license for full licensing information.
+ *
+ * In applying this license CERN does not waive the privileges and immunities
+ * granted to it by virtue of its status as an Intergovernmental Organization
+ * or submit itself to any jurisdiction.
+ */
+
+const Joi = require('joi');
+const { emptyDTO, tokenDTO } = require('./commons.dto');
+
+const emptyDataRequestDTO = Joi.object({
+    query: emptyDTO,
+    params: emptyDTO,
+    body: emptyDTO,
+});
+
+/**
+ * DTO that allows only:
+ * query.token - webUi token
+ * query.filter - object for building sequelize where-cluase
+ */
+const stdDataRequestDTO = Joi.object({
+    query: tokenDTO.keys({ filter: Joi.object({}).unknown(true) }), //TODO make more strict
+    params: emptyDTO,
+    body: emptyDTO,
+});
+
+module.exports = {
+    stdDataRequestDTO,
+    emptyDataRequestDTO,
+};

app/lib/server/controllers/ApiDocumentation.controller.js

@@ -11,8 +11,8 @@
  * or submit itself to any jurisdiction.
  */
 
-const { STDEntityDTO } = require('../../domain/dtos');
-const { validateDTO } = require('../utilities');
+const { emptyDataRequestDTO } = require('../../domain/dtos');
+const { validateDtoOrRepondOnFailure } = require('../utilities');
 
 const getApiDocsAsJson = (routes) => routes.map(({ method, path, description }) => ({ method, path, description }));
 
@@ -33,15 +33,13 @@ class ApiDocumentationCotroller {
      * Express hanlder for whole api description request
      */
     async getDocsHandler(req, res) {
-        const validatedDTO = await validateDTO(STDEntityDTO, req, res);
-        if (!validatedDTO) {
-            return;
+        const validatedDTO = await validateDtoOrRepondOnFailure(emptyDataRequestDTO, req, res);
+        if (validatedDTO) {
+            if (!this.apiDocsJson) {
+                res.status(500).json({ message: 'Server misconfigured, please, contact administrator' });
+            }
+            res.json(this.apiDocsJson);
         }
-
-        if (!this.apiDocsJson) {
-            res.status(500).json({ message: 'Server misconfigured, please, contact administrator' });
-        }
-        res.json(this.apiDocsJson);
     }
 }
 

app/lib/server/controllers/period.controller.js

@@ -12,8 +12,8 @@
  */
 
 const { periodService } = require('../../services/periods/PeriodService');
-const { STDEntityDTO } = require('../../domain/dtos');
-const { validateDTO } = require('../utilities');
+const { stdDataRequestDTO } = require('../../domain/dtos');
+const { validateDtoOrRepondOnFailure } = require('../utilities');
 
 /**
  * List All runs in db
@@ -23,7 +23,7 @@ const { validateDTO } = require('../utilities');
  * @returns {undefined}
  */
 const listPeriodsHandler = async (req, res, next) => {
-    const validatedDTO = await validateDTO(STDEntityDTO, req, res);
+    const validatedDTO = await validateDtoOrRepondOnFailure(stdDataRequestDTO, req, res);
     if (validatedDTO) {
         const runs = await periodService.getAll(validatedDTO.query);
         res.json({

app/lib/server/controllers/run.controller.js

@@ -12,8 +12,8 @@
  */
 
 const { runService } = require('../../services/runs/RunService');
-const { STDEntityDTO } = require('../../domain/dtos');
-const { validateDTO } = require('../utilities');
+const { stdDataRequestDTO } = require('../../domain/dtos');
+const { validateDtoOrRepondOnFailure } = require('../utilities');
 
 /**
  * List All runs in db
@@ -22,8 +22,9 @@ const { validateDTO } = require('../utilities');
  * @param {Object} next express next handler
  * @returns {undefined}
  */
+
 const listRunsHandler = async (req, res, next) => {
-    const validatedDTO = await validateDTO(STDEntityDTO, req, res);
+    const validatedDTO = await validateDtoOrRepondOnFailure(stdDataRequestDTO, req, res);
     if (validatedDTO) {
         const runs = await runService.getAll(validatedDTO.query);
         res.json({

app/lib/server/routers/index.js

@@ -11,6 +11,7 @@
  * or submit itself to any jurisdiction.
  */
 
+const { controllerHandlerWrapper } = require('../utilities');
 const periodRouter = require('./period.router.js');
 const runRouter = require('./run.router.js');
 const docsRouter = require('./docs.router.js');
@@ -41,7 +42,7 @@ function buildRoute(controllerTree) {
             if (process.env.ENV_MODE === 'test' || process.env.ENV_MODE === 'dev') {
                 args.public = true;
             }
-            routesStack.push({ method, path, controller, args, description });
+            routesStack.push({ method, path, controller: controllerHandlerWrapper(controller), args, description });
         } else if (method && ! controller || ! method && controller) {
             throw `Routers incorrect configuration for ${path}, [method: '${method}'], [controller: '${controller}']`;
         }

app/lib/domain/dtos/STDEntity.dto.js → app/lib/server/utilities/controllerWrapper.js

@@ -11,13 +11,11 @@
  * or submit itself to any jurisdiction.
  */
 
-const Joi = require('joi');
-const { emptyDTO, tokenDTO } = require('./commons.dto');
+const controllerHandlerWrapper = (controllerHandler) =>
+    async (req, res, next) =>
+        await controllerHandler(req, res, next)
+            .catch((err) => res.status(400).json({ error: err.message }));
 
-const STDEntityDTO = Joi.object({
-    query: tokenDTO, //TODO extend with filters
-    params: emptyDTO,
-    body: emptyDTO,
-});
-
-module.exports = STDEntityDTO;
+module.exports = {
+    controllerHandlerWrapper,
+};

app/lib/server/utilities/dtoValidation.js

@@ -21,7 +21,7 @@
  *                          HTTP request.
  * @returns {Promise<object>} Returns the validated value or null if an error occurred.
  */
-const validateDTO = async (dto, req, res) => {
+const validateDtoOrRepondOnFailure = async (dto, req, res) => {
     const { query, params, body } = req;
     try {
         return await dto.validateAsync({ query, params, body }, {
@@ -40,5 +40,5 @@ const validateDTO = async (dto, req, res) => {
 };
 
 module.exports = {
-    validateDTO,
+    validateDtoOrRepondOnFailure,
 };

app/lib/server/utilities/filterParser.js

@@ -0,0 +1,187 @@
+/**
+ * @license
+ * Copyright CERN and copyright holders of ALICE O2. This software is
+ * distributed under the terms of the GNU General Public License v3 (GPL
+ * Version 3), copied verbatim in the file "COPYING".
+ *
+ * See http://alice-o2.web.cern.ch/license for full licensing information.
+ *
+ * In applying this license CERN does not waive the privileges and immunities
+ * granted to it by virtue of its status as an Intergovernmental Organization
+ * or submit itself to any jurisdiction.
+ */
+
+const { Op } = require('sequelize');
+const { throwWrapper } = require('../../utils');
+
+// eslint-disable-next-line max-len
+const unitaryRelationalOperators = new Set(['gt', 'gte', 'lt', 'lte', 'ne', 'like', 'notLike', 'iLike', 'notILike', 'regexp', 'notRegexp', 'iRegexp', 'notIRegexp']);
+const arrayRelationalConditions = new Set(['between', 'notBetween', 'in', 'notIn']);
+const relationalOperators = new Set([...arrayRelationalConditions, ...unitaryRelationalOperators]);
+const logicalOperators = new Set(['not', 'and', 'or']);
+
+const _groupOperatorName = '_goperator';
+const defaultGroupOperator = 'and';
+
+const reservedNames = new Set([...relationalOperators, ...logicalOperators, ...[_groupOperatorName]]);
+
+const transformationSentinel = 'and';
+
+class TransformHelper {
+    constructor(opts) {
+        this.opts = opts;
+    }
+
+    insertSentinel(filter) {
+        return { [transformationSentinel]: filter };
+    }
+
+    removeSentinel(filter) {
+        return filter[Op[transformationSentinel]];
+    }
+
+    handleIntermidiateFilterNode(group, groupOperator) {
+        const transformedSubfilter = this.transformHelper(group);
+        return this.opts.pruneRedundantANDOperator && groupOperator === defaultGroupOperator ?
+            transformedSubfilter :
+            { [Op[groupOperator]]: transformedSubfilter };
+    }
+
+    handleArrayValues(val) {
+        return val.split(/,/).map((v) => v.trim());
+    }
+
+    handelFieldFilterTail(fieldGroup, groupOperator) {
+        const transformedFieldGroup = Object.fromEntries(Object.entries(fieldGroup)
+            .map(([relOp, val]) => [
+                Op[relOp] ?? throwWrapper(new Error(`No relational operator <${relOp}>, only <${[...relationalOperators]}> are allowed`)),
+                arrayRelationalConditions.has(relOp) ? this.handleArrayValues(val) : val,
+            ]));
+        return this.opts.pruneRedundantANDOperator && groupOperator === defaultGroupOperator ?
+            transformedFieldGroup :
+            { [Op[groupOperator]]: transformedFieldGroup };
+    }
+
+    pullGroupOperator(group) {
+        const readGroupOperator = group[_groupOperatorName];
+        const groupOperator = readGroupOperator ?? defaultGroupOperator;
+        delete group[_groupOperatorName];
+        return groupOperator;
+    }
+
+    transformHelper(filter) {
+        return Object.fromEntries(Object.entries(filter)
+            .map(([k, group]) => {
+                const groupOperator = this.pullGroupOperator(group);
+                if (!reservedNames.has(k)) { // Assumes that k is field from db view
+                    if (typeof group === 'object') {
+                        return [k, this.handelFieldFilterTail(group, groupOperator)];
+                    } else {
+                        return [k, group]; // Assumes that there is not relation operator
+                    }
+                } else { // Then k stands for logical operator
+                    return [
+                        Op[k] ?? throwWrapper(new Error(`No logical operator <${k}>, only <${[...logicalOperators]}> are allowed`)),
+                        this.handleIntermidiateFilterNode(group, groupOperator),
+                    ];
+                }
+            }));
+    }
+
+    transform(filter) {
+        if (!filter) {
+            return {};
+        }
+        filter = this.insertSentinel(filter);
+        filter = this.transformHelper(filter);
+        filter = this.removeSentinel(filter);
+        return filter;
+    }
+}
+
+/**
+ * Transform filter object from http request.query to sequelize where-clause object
+ * Considering the filter object is a tree, each root -> leaf path can be described as:
+ * 1. one of the two patterns
+ * (dots separate nodes, uppercase written words are non-terminal (as in formal gramatics terminology)
+ * and that lowercase are terminal, e.g. operator 'not' is terminal):
+ *
+ *   (.LOGIC_OPERATR)*.DB_FIELD_NAME.(RELATION_OPERATOR.VALUE | GROUP_OPERATOR.(LOGIC_OPERATOR)) | or
+ *   (.LOGIC_OPERATR)*.GROUP_OPERATOR.(LOGIC_OPERATOR)
+ *
+ *      where '_goperator' is the only GROUP_OPERATOR terminal,
+ *
+ * So each top-down path begins with sequence of logical operators and ends with
+ *         DB_FIELD_NAME._fgoperator.(and|or|not)
+ * or with DB_FIELD_NAME.RELATIONAL_OPERATOR.VALUE
+ * or with ._fgoperator.(and|or|not)
+ *
+ * 2. one of the two corresnponding regexes:
+ *   (\.((and)|(or)|(not)))*\.([_a-z][_a-z0-9]*)\.(((and)|(or)|(not))\..*)|(_fgoperator\.((or)|(and)|(not))))                |
+ *   -----------------------  ------------------   -----------------------|--------------------------------                  | or
+ *       (.LOGIC_OPERATR)*      .DB_FIELD_NAME     RELATION_OPERATOR.VALUE   GROUP_OPERATOR.LOGIC_OPERATOR                   |
+ *
+ *   (\.((and)|(or)|(not)))*\._fgoperator\.((or)|(and)|(not))
+ *   -----------------------  --------------------------------
+ *       (.LOGIC_OPERATR)*     .GROUP_OPERATOR.LOGIC_OPERATOR
+ *
+ *
+ * 3. As this api is intended for handling http requests in which filtering is defined by url query params,
+ * the GROUP_OPERATOR is syntax sugar:
+ * Instead of /?filter[field1][or][gt]=10&filter[field1][or][lt]=3
+ * it can be
+ *            /?filter[field1][gt]=10&filter[field][lt]=3&filter[field1][_fgoperator]=or
+ * GROUP_OPERATOR can be ommited, its default value is an 'and'.
+ *
+ * Example:
+ * 1` URL is: .../?filter[or][field1][gt]=10&filter[or][field1][lt]=3&filter[or][field1][_goperator]=or ...
+ *              ... &filter[or][filed2][like]=LHC_%pass&filter[or][filed2][notLike]=LHC_c%
+ * 2` The url will be parsed by qs in express to:
+ * const query = {
+ * "filter": {
+ *   "or": {
+ *     "field1": {
+ *       "gt": "10",
+ *       "lt": "3",
+ *       "_goperator": "or"
+ *     },
+ *     "filed2": {
+ *       "like": "LHC_%pass",
+ *       "notLike": "LHC_c%"
+ *     }
+ *   }
+ * }
+ *
+ * 3` query.filter is being passed as an argument to filterToSequelizeWhereClause function
+ * which returns sequelize-compatible where-clause
+ *
+ * {
+ *  [Symbol(or)]: {
+ *    field1: {
+ *      [Symbol(or)]: {
+ *        [Symbol(gt)]: '10',
+ *        [Symbol(lt)]: '3'
+ *      }
+ *    },
+ *    filed2: {
+ *      [Symbol(and)]: {
+ *        [Symbol(like)]: 'LHC_%pass',
+ *        [Symbol(notLike)]: 'LHC_c%'
+ *      }
+ *    }
+ *  }
+ * }
+ *
+ * What is equivalent to sql:
+ * WHERE (field1 > 10 OR field1 < 3) OR (field2 like 'LHC_%pass' AND field2 NOT LIKE 'LHC_c%')
+ *
+ * @param {Object} filter - from req.query
+ * @param {boolean} pruneRedundantANDOperator - if true (default) remove unnecessaary 'and' operators
+ * @returns {Object} sequelize where object
+ */
+const filterToSequelizeWhereClause = (filter, pruneRedundantANDOperator = true) =>
+    new TransformHelper({ pruneRedundantANDOperator }).transform(filter);
+
+module.exports = {
+    filterToSequelizeWhereClause,
+};

app/lib/server/utilities/index.js

@@ -12,7 +12,11 @@
  */
 
 const dtoValidation = require('./dtoValidation');
+const filterParser = require('./filterParser');
+const controllerWrapper = require('./controllerWrapper');
 
 module.exports = {
     ...dtoValidation,
+    ...filterParser,
+    ...controllerWrapper,
 };