Skip to content
This repository was archived by the owner on Jan 14, 2025. It is now read-only.

Filtering #123

Merged
Ehevi merged 8 commits into from
Jul 14, 2023
Merged

Filtering #123

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
b8e4df7
backend adjust for coplex filtering
xsalonx Jul 13, 2023
46081b0
filter adding to url
xsalonx Jul 13, 2023
eccbb67
stash with backend changes for filtering
xsalonx Jul 13, 2023
d6c065b
between instead of from/to on frontend
xsalonx Jul 13, 2023
528d515
preprocessing for filtering
xsalonx Jul 13, 2023
719872f
filtering on backend done
xsalonx Jul 13, 2023
ce751f3
Merge branch 'master' into filtering
xsalonx Jul 14, 2023
e08ae86
cleanup
xsalonx Jul 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions app/config/public.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,13 @@
* or submit itself to any jurisdiction.
*/

const matchExcludeType = 'matchexclude';
const fromToType = 'fromto';

const { roles, flags: flagsTypes, detectors } = require('./rct-data');

module.exports = { // Properties that will be provided to frontend in the public folder
filterTypes: {
matchExcludeType,
fromToType,
match: 'match',
exclude: 'exclude',
between: 'between',
},

roles,
Expand Down
168 changes: 61 additions & 107 deletions app/lib/database/QueryBuilder.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@
const config = require('../config/configProvider.js');
const views = require('./views');
const procedures = require('./procedures')
const { adjustValuesToSql, switchCase } = require('../utils');

const { pageNames: PN, procedures: PC } = config.public;
const { pageNames: PN, procedures: PC, filterTypes } = config.public;
const DRP = config.public.dataReqParams;

const pageToViewName = {};
Expand All @@ -32,129 +33,83 @@ pageToViewName[PN.flags] = 'flags_view'
* Class responsible for parsing url params, payloads of client request to sql queries
*/


const filterTypes = ['match', 'exclude', 'from', 'to'];

const ops = {
NOTLIKE: 'NOT LIKE',
LIKE: 'LIKE',
IN: 'IN',
NOTIN: 'NOT IN',
FROM: '>=',
TO: '<=',
EQ: '==',
NE: '!=',
AND: 'AND',
OR: 'OR',
};

const filtersTypesToSqlOperand = {
match: 'LIKE',
exclude: 'NOT LIKE',
from: '>=',
to: '<='
}

const logicalOperandsPerFilters = {
match: {
string: ops.LIKE,
number: ops.IN,
},
exclude: {
string: ops.NOTLIKE,
number: ops.NOTIN,
},
from: ops.FROM,
to: ops.TO,
};

const filtersTypesToSqlValueQuoted = {
match: '\'',
exclude: '\'',
from: '',
to: ''
const handleBetween = (fieldName, pairsList) => {
if (! Array.isArray(pairsList)) {
pairsList = [pairsList]
}
return pairsList.map((p) => {
const value = p.split(',');
const [left, right] = adjustValuesToSql(value);
if (value[0] && value[1]) {
return `${fieldName} BETWEEN ${left} AND ${right}`;
} else if (value[0]) {
return `${fieldName} >= ${left}`;
} else if (value[1]) {
return `${fieldName} <= ${right}`;
}
}).join(' OR ');
}

//match take precedens
const controlForNoArrays = {
notarray: {
match: {
string: [ops.LIKE, ops.OR],
number: [ops.EQ, ops.OR],
},
exclude: {
string: [ops.NOTLIKE, ops.AND],
number: [ops.NE, ops.AND],
},
from: [ops.FROM, ops.AND],
to: [ops.TO, ops.AND],
},

array: {
match: {
string: ops.LIKE,
number: ops.EQ,
},
exclude: {
string: ops.NOTLIKE,
number: ops.NE,
},
from: ops.FROM,
to: ops.TO,
},
const handleLike = (fieldName, values, like) => {
if (! Array.isArray(values)) {
values = [values]
}
values = values.map((subv) => subv.split(',').map((v) => v.trim()).filter((v) => v)).flat()

const notS = ! like ? 'NOT' : '';
if (values.every((v) => isNaN(v))) {
// assume that it is for patterns
const valuesPattern = values.join('|');
return `${fieldName} ${notS} SIMILAR TO '${valuesPattern}'`;
} else if (values.every((v) => ! isNaN(v))) {
// assumes that it is for numbers with 'in' clause
const valuesList = values.join(',');
return `${fieldName} ${notS} IN (${valuesList})`;
}
}


class QueryBuilder {

static filteringPart(params) {
const filtersTypesToParams = {
match: [],
exclude: [],
from: [],
to: []
between: []
}

// Mapping search params to categorized { key, value } pairs
const filterTypesRegex= new RegExp(filterTypes.map((t) => `(.*-${t})`).join('|'));
const filterParams = Object.entries(params).filter(([k, v]) => k.match(filterTypesRegex));

for (let [filedNameAndFilterType, value] of Object.entries(filterParams)) {
const [fieldName, filterType] = filedNameAndFilterType.split('-');
if (! Array.isArray(value) && /.*,.*/.test(value)) {
value = value.split(',').map((s) => s.trim())
}
if (filterType in filtersTypesToParams) {
filtersTypesToParams[filterType].push({ fieldName, value })
}
}

// console.log(filtersTypesToParams)

// Object.entries(filtersTypesToParams).map(([t, pli]) => {
// pli.map(([fN, fv]) => {
// if (t
// })
// })
const filterTypesRegex= new RegExp(Object.keys(filterTypes).map((t) => `(.*-${t})`).join('|'));
const filterParams = Object.entries(params)
.filter(([k, v]) => k.match(filterTypesRegex))
.map(([k, v]) => [...k.split('-'), v]);
const fields2Filters = {};
filterParams.forEach((l) => fields2Filters[l[0]] = [])
filterParams.forEach((l) => fields2Filters[l[0]].push(l.slice(1)))

// Joining previous to sql clause
const sqlWhereClause = Object.keys(filtersTypesToParams)
.map((t) => {
const qt = filtersTypesToSqlValueQuoted[t];
const operand = filtersTypesToSqlOperand[t];
return filtersTypesToParams[t]
.map(({ fieldName, value }) => `"${fieldName}" ${operand} ${qt}${value}${qt}`)
.join("AND");})
.filter((clause) => clause?.length > 0)
.join("AND");


const cll = Object.entries(fields2Filters).map(([fieldName, clauses]) => {
return Object.fromEntries(clauses.map((cl) => {
const [filterType, values] = cl;
console.log(filterType, values)
const cases = {
'between': () => handleBetween(fieldName, values),
'match': () => handleLike(fieldName, values, true),
'exclude': () => handleLike(fieldName, values, false),
}
return[filterType, switchCase(filterType, cases, { default: () => '' })()];
}))

})

const sqlWhereClause = cll.map((cl) => { // for each field
const mbpart = [cl.match, cl.between].filter((v) => v).join(' OR \n');
const expart = cl.exclude ? ' AND ' + cl.exclude : ''
return `${mbpart ? '(' + mbpart + ')' : ''} ${expart}`;
}).join(' AND \n');

return sqlWhereClause?.length > 0 ? `WHERE ${sqlWhereClause}` : '';
}

static buildSelect(params) {
// console.log(params)
delete params.aaa;

const dataSubsetQueryPart = (params) => params[DRP.countRecords] === 'true' ? '' :
`LIMIT ${params[DRP.rowsOnSite]} OFFSET ${params[DRP.rowsOnSite] * (params[DRP.site] - 1)}`;
Expand Down Expand Up @@ -183,7 +138,6 @@ class QueryBuilder {
${QueryBuilder.filteringPart(params)}
${orderingPart(params)}
${dataSubsetQueryPart(params)};`;
// console.log(a);
return a;
}

Expand Down
2 changes: 1 addition & 1 deletion app/lib/utils/sql-utils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ function adjustValuesToSql(v) {
if (typeof v == 'string') {
if (v.length == 0) {
return null;
} else if (! sqlValueKeywords.includes(v?.trim().toUpperCase())) {
} else if (! sqlValueKeywords.includes(v?.trim().toUpperCase()) && isNaN(v)) {
return `'${v}'`;
}
}
Expand Down
89 changes: 49 additions & 40 deletions app/public/views/userView/data/table/filtering/filter.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/* eslint-disable no-console */
/**
* @license
* Copyright 2019-2020 CERN and copyright holders of ALICE O2.
Expand All @@ -14,60 +15,68 @@

import { h } from '/js/src/index.js';
import { getFieldName } from '../../headersSpecials.js';
import { RCT } from '../../../../../config.js';
const { filterTypes } = RCT;

export default function filter(model) {
const data = model.getCurrentData();
const { fields } = data;

function onFilteringTypeChange() {
const filteringTypeSelect = document.getElementById('showOptionsType');
const selectedValue = filteringTypeSelect.options[filteringTypeSelect.selectedIndex].value;
const types = selectedValue.split('-');
const filteringTypeSelect = document.getElementById('filters-opts-select');
const selectedType = filteringTypeSelect.options[filteringTypeSelect.selectedIndex].value;
const types = selectedType == filterTypes.between ? ['from', 'to'] : [filterTypes.match, filterTypes.exclude];

const matchFromPlaceholder = document.getElementById('match-from-placeholder');
const excludeToPlaceholder = document.getElementById('exclude-to-placeholder');
[matchFromPlaceholder.innerHTML, excludeToPlaceholder.innerHTML] = types;
const leftFilterPlaceholder = document.getElementById('left-filter-placeholder');
const rightFilterPlaceholder = document.getElementById('right-filter-placeholder');
[leftFilterPlaceholder.innerHTML, rightFilterPlaceholder.innerHTML] = types;
}

function onFilterSubmit() {
const fieldNameSelect = document.getElementById('showOptionsField');
const filteringTypeSelect = document.getElementById('filters-opts-select');
const selectedType = filteringTypeSelect.options[filteringTypeSelect.selectedIndex].value;

const fieldNameSelect = document.getElementById('show-options-field');
const fieldNameValue = fieldNameSelect.options[fieldNameSelect.selectedIndex].value;

const matchFromType = document.getElementById('match-from-placeholder').innerHTML;
const excludeToType = document.getElementById('exclude-to-placeholder').innerHTML;
const leftFilterType = document.getElementById('left-filter-placeholder').innerHTML;
const rightFilterType = document.getElementById('right-filter-placeholder').innerHTML;

const matchFromInput = document.getElementById('match-from-input').value;
const excludeToInput = document.getElementById('exclude-to-input').value;
const leftFilterInput = document.getElementById('left-filter-input').value;
const rightFilterInput = document.getElementById('right-filter-input').value;

// eslint-disable-next-line no-console
console.log(fieldNameValue);
// eslint-disable-next-line no-console
console.log(matchFromType);
// eslint-disable-next-line no-console
console.log(matchFromInput);
// eslint-disable-next-line no-console
console.log(excludeToInput);

let matchFromPhrase = '';
let excludeToPhrase = '';
if (matchFromInput) {
matchFromPhrase = `${encodeURI(fieldNameValue)}-${encodeURI(matchFromType)}=${encodeURI(matchFromInput)}`;
}
if (excludeToInput) {
excludeToPhrase = `${encodeURI(fieldNameValue)}-${encodeURI(excludeToType)}=${encodeURI(excludeToInput)}`;
console.log({
fieldNameValue,

rightFilterType,
rightFilterInput,

leftFilterType,
leftFilterInput,
});

let filterPhrase = '';
if (selectedType == filterTypes.between) {
// eslint-disable-next-line max-len
filterPhrase += `&${encodeURI(fieldNameValue)}-${encodeURI(filterTypes.between)}=${encodeURI(leftFilterInput)},${encodeURI(rightFilterInput)}`;
} else {
// So Match or Exclude filtering type
// eslint-disable-next-line max-len
filterPhrase += leftFilterInput ? `&${encodeURI(fieldNameValue)}-${encodeURI(leftFilterType)}=${encodeURI(leftFilterInput)}` : '';
filterPhrase += rightFilterInput ? `&${encodeURI(fieldNameValue)}-${encodeURI(rightFilterType)}=${encodeURI(rightFilterInput)}` : '';
}

const url = model.router.getUrl();
const newUrl = new URL(`${url.href}`
+ `${matchFromPhrase ? `&${matchFromPhrase}` : ''}`
+ `${excludeToPhrase ? `&${excludeToPhrase}` : ''}`);

const newUrl = new URL(`${url.href}${filterPhrase}`);
// eslint-disable-next-line no-console
console.log(newUrl);

// Model.router.go(newUrl);
model.router.go(newUrl);
}

const aggregatedFiltersTypes = `${filterTypes.match}-${filterTypes.exclude}`;

return h('.font-size-small', [
h('div.flex-wrap.justify-between.items-center',
h('div.flex-wrap.justify-between.items-center',
Expand All @@ -78,46 +87,46 @@ export default function filter(model) {
h('div.flex-wrap.justify-between.items-center',
h('div.flex-wrap.justify-between.items-center',
h('select.select.filter-select', {
id: 'showOptionsField',
id: 'show-options-field',
name: 'showOptions' }, [
h('option', { value: '', selected: true, disabled: true, hidden: true }, 'Field'),
fields.filter((field) => getFieldName(model, field))
.map((field) => h('option', { value: field.name }, getFieldName(model, field))),
], h('.close-10')),

h('select.select.filter-select', {
id: 'showOptionsType',
id: 'filters-opts-select',
name: 'showOptions',
onchange: () => onFilteringTypeChange(),
}, [
h('option', { value: '', selected: true, disabled: true, hidden: true }, 'Filtering type'),
h('option', { value: 'match-exclude' }, 'match-exclude'),
h('option', { value: 'from-to' }, 'from-to'),
h('option', { value: aggregatedFiltersTypes }, aggregatedFiltersTypes),
h('option', { value: filterTypes.between }, filterTypes.between),
], h('.close-10')),

h('.text-field',
h('input.form-control.relative', {
style: 'width:120px',
type: 'text',
value: document.getElementById('showOptionsType')?.options[Selection.selectedIndex]?.value,
value: document.getElementById('filters-opts-select')?.options[Selection.selectedIndex]?.value,
disabled: false,
id: 'match-from-input',
id: 'left-filter-input',
required: true,
// TODO: pattern
}),
h('span.placeholder', { id: 'match-from-placeholder' }, 'match/from')),
h('span.placeholder', { id: 'left-filter-placeholder' }, 'match/from')),

h('.text-field',
h('input.form-control.relative', {
style: 'width:120px',
type: 'text',
value: '',
disabled: false,
id: 'exclude-to-input',
id: 'right-filter-input',
required: true,
// TODO: pattern
}),
h('span.placeholder', { id: 'exclude-to-placeholder' }, 'to/exclude')),
h('span.placeholder', { id: 'right-filter-placeholder' }, 'to/exclude')),

h('button.btn.btn-primary', {
onclick: () => onFilterSubmit(),
Expand Down