From a1e335d41f965dea18c80c5cc09d7b933bf91780 Mon Sep 17 00:00:00 2001 From: Maha Benzekri Date: Fri, 13 Dec 2024 14:28:22 +0100 Subject: [PATCH] test --- .github/workflows/tests.yaml | 64 +++++++++---------- bin/list_bucket_metrics.js | 2 +- bin/list_metrics.js | 2 +- lib/Config.js | 6 +- lib/api/multiObjectDelete.js | 1 - lib/api/objectDelete.js | 1 - lib/routes/utilities/pushReplicationMetric.js | 2 +- lib/utapi/utapi.js | 4 +- lib/utapi/utapiReindex.js | 20 +++--- lib/utapi/utapiReplay.js | 14 ++-- lib/utapi/utilities.js | 32 ++++++---- package.json | 16 ++--- tests/unit/Config.js | 4 +- tests/utapi/utilities.js | 2 +- tests/utilities/mock/index.js | 1 - 15 files changed, 86 insertions(+), 85 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 16b560a8b9..b58a92d289 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -342,38 +342,38 @@ jobs: source: /tmp/artifacts if: always() - utapi-v2-tests: - runs-on: ubuntu-latest - needs: build - env: - ENABLE_UTAPI_V2: t - S3BACKEND: mem - BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket - CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }} - MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} - JOB_NAME: ${{ github.job }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup CI environment - uses: ./.github/actions/setup-ci - - name: Setup CI services - run: docker compose up -d - working-directory: .github/docker - - name: Run file utapi v2 tests - run: |- - set -ex -o pipefail; - bash wait_for_local_port.bash 8000 40 - yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log - - name: Upload logs to artifacts - uses: scality/action-artifacts@v4 - with: - method: upload - url: https://artifacts.scality.net - user: ${{ secrets.ARTIFACTS_USER }} - password: ${{ secrets.ARTIFACTS_PASSWORD }} - source: /tmp/artifacts - if: always() + # utapi-v2-tests: + # runs-on: ubuntu-latest + # needs: build + # env: + # ENABLE_UTAPI_V2: t + # S3BACKEND: mem + # BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket + # CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }} + # MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} + # JOB_NAME: ${{ github.job }} + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: Setup CI environment + # uses: ./.github/actions/setup-ci + # - name: Setup CI services + # run: docker compose up -d + # working-directory: .github/docker + # - name: Run file utapi v2 tests + # run: |- + # set -ex -o pipefail; + # bash wait_for_local_port.bash 8000 40 + # yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log + # - name: Upload logs to artifacts + # uses: scality/action-artifacts@v4 + # with: + # method: upload + # url: https://artifacts.scality.net + # user: ${{ secrets.ARTIFACTS_USER }} + # password: ${{ secrets.ARTIFACTS_PASSWORD }} + # source: /tmp/artifacts + # if: always() quota-tests: runs-on: ubuntu-latest diff --git a/bin/list_bucket_metrics.js b/bin/list_bucket_metrics.js index e97aefebc5..190bef32bf 100644 --- a/bin/list_bucket_metrics.js +++ b/bin/list_bucket_metrics.js @@ -1,4 +1,4 @@ #!/usr/bin/env node -'use strict'; // eslint-disable-line strict +'use strict'; require('../lib/utapi/utilities.js').listMetrics('buckets'); diff --git a/bin/list_metrics.js b/bin/list_metrics.js index c46c5c22ff..be8c71ee1c 100644 --- a/bin/list_metrics.js +++ b/bin/list_metrics.js @@ -1,4 +1,4 @@ #!/usr/bin/env node -'use strict'; // eslint-disable-line strict +'use strict'; require('../lib/utapi/utilities.js').listMetrics(); diff --git a/lib/Config.js b/lib/Config.js index 696b69abde..fee1974edf 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -16,7 +16,7 @@ const validExternalBackends = require('../constants').externalBackends; const { azureAccountNameRegex, base64Regex, allowedUtapiEventFilterFields, allowedUtapiEventFilterStates, } = require('../constants'); -const { utapiVersion } = require('utapi'); +// const { utapiVersion } = require('utapi'); const { scaleMsPerDay } = s3middleware.objectUtils; const constants = require('../constants'); @@ -1220,7 +1220,7 @@ class Config extends EventEmitter { maxStaleness, enableInflights, }; - if (config.utapi) { + if (config.utapi && false) { this.utapi = { component: 's3' }; if (config.utapi.host) { assert(typeof config.utapi.host === 'string', @@ -1353,7 +1353,7 @@ class Config extends EventEmitter { } } if (Object.keys(this.locationConstraints).some( - loc => this.locationConstraints[loc].sizeLimitGB)) { + loc => this.locationConstraints[loc].sizeLimitGB) && false) { assert(this.utapi && this.utapi.metrics && this.utapi.metrics.includes('location'), 'bad config: if storage size limit set on a location ' + diff --git a/lib/api/multiObjectDelete.js b/lib/api/multiObjectDelete.js index 1ee4a10dab..930a295f3e 100644 --- a/lib/api/multiObjectDelete.js +++ b/lib/api/multiObjectDelete.js @@ -349,7 +349,6 @@ function getObjMetadataAndDelete(authInfo, canonicalID, request, options.doesNotNeedOpogUpdate = true; } if (objMD.uploadId) { - // eslint-disable-next-line options.replayId = objMD.uploadId; } return services.deleteObject(bucketName, objMD, diff --git a/lib/api/objectDelete.js b/lib/api/objectDelete.js index ea5e2f2070..35161e13e1 100644 --- a/lib/api/objectDelete.js +++ b/lib/api/objectDelete.js @@ -1,4 +1,3 @@ -/* eslint-disable indent */ const async = require('async'); const { errors, versioning, s3middleware } = require('arsenal'); const checkDateModifiedHeaders = s3middleware.checkDateModifiedHeaders; diff --git a/lib/routes/utilities/pushReplicationMetric.js b/lib/routes/utilities/pushReplicationMetric.js index b82b043a08..c81027a494 100644 --- a/lib/routes/utilities/pushReplicationMetric.js +++ b/lib/routes/utilities/pushReplicationMetric.js @@ -18,7 +18,7 @@ function getMetricToPush(prevObjectMD, newObjectMD) { prevObjectMD.getTags(), newObjectMD.getTags() ); - } catch (e) { + } catch { return 'replicateTags'; } return null; diff --git a/lib/utapi/utapi.js b/lib/utapi/utapi.js index 83c6925540..f2b6db06e5 100644 --- a/lib/utapi/utapi.js +++ b/lib/utapi/utapi.js @@ -1,9 +1,9 @@ require('werelogs').stderrUtils.catchAndTimestampStderr(); const _config = require('../Config').config; -const { utapiVersion, UtapiServer: utapiServer } = require('utapi'); +// const { utapiVersion, UtapiServer: utapiServer } = require('utapi'); // start utapi server -if (utapiVersion === 1 && _config.utapi) { +if (utapiVersion === 1 && _config.utapi && false) { const fullConfig = Object.assign({}, _config.utapi, { redis: _config.redis }); if (_config.vaultd) { diff --git a/lib/utapi/utapiReindex.js b/lib/utapi/utapiReindex.js index 9bc1522355..004bcbd525 100644 --- a/lib/utapi/utapiReindex.js +++ b/lib/utapi/utapiReindex.js @@ -1,11 +1,11 @@ -require('werelogs').stderrUtils.catchAndTimestampStderr(); -const UtapiReindex = require('utapi').UtapiReindex; -const { config } = require('../Config'); +// require('werelogs').stderrUtils.catchAndTimestampStderr(); +// const UtapiReindex = require('utapi').UtapiReindex; +// const { config } = require('../Config'); -const reindexConfig = config.utapi && config.utapi.reindex; -if (reindexConfig && reindexConfig.password === undefined) { - reindexConfig.password = config.utapi && config.utapi.redis - && config.utapi.redis.password; -} -const reindex = new UtapiReindex(reindexConfig); -reindex.start(); +// const reindexConfig = config.utapi && config.utapi.reindex; +// if (reindexConfig && reindexConfig.password === undefined) { +// reindexConfig.password = config.utapi && config.utapi.redis +// && config.utapi.redis.password; +// } +// const reindex = new UtapiReindex(reindexConfig); +// reindex.start(); diff --git a/lib/utapi/utapiReplay.js b/lib/utapi/utapiReplay.js index e958d83da0..5f220d1312 100644 --- a/lib/utapi/utapiReplay.js +++ b/lib/utapi/utapiReplay.js @@ -1,8 +1,8 @@ -require('werelogs').stderrUtils.catchAndTimestampStderr(); -const UtapiReplay = require('utapi').UtapiReplay; -const _config = require('../Config').config; +// require('werelogs').stderrUtils.catchAndTimestampStderr(); +// const UtapiReplay = require('utapi').UtapiReplay; +// const _config = require('../Config').config; -const utapiConfig = _config.utapi && - Object.assign({}, _config.utapi, { redis: _config.redis }); -const replay = new UtapiReplay(utapiConfig); // start utapi server -replay.start(); +// const utapiConfig = _config.utapi && +// Object.assign({}, _config.utapi, { redis: _config.redis }); +// const replay = new UtapiReplay(utapiConfig); // start utapi server +// replay.start(); diff --git a/lib/utapi/utilities.js b/lib/utapi/utilities.js index b29ed172bd..162b2b5b37 100644 --- a/lib/utapi/utilities.js +++ b/lib/utapi/utilities.js @@ -2,27 +2,27 @@ const http = require('http'); const https = require('https'); const commander = require('commander'); const { auth } = require('arsenal'); -const { UtapiClient, utapiVersion } = require('utapi'); +// const { UtapiClient, utapiVersion } = require('utapi'); const logger = require('../utilities/logger'); const _config = require('../Config').config; const { suppressedUtapiEventFields: suppressedEventFields } = require('../../constants'); // setup utapi client let utapiConfig; -if (utapiVersion === 1 && _config.utapi) { - if (_config.utapi.redis === undefined) { - utapiConfig = Object.assign({}, _config.utapi, { redis: _config.redis }); - } else { - utapiConfig = Object.assign({}, _config.utapi); - } -} else if (utapiVersion === 2) { - utapiConfig = Object.assign({ - tls: _config.https, - suppressedEventFields, - }, _config.utapi || {}); -} +// if (utapiVersion === 1 && _config.utapi) { +// if (_config.utapi.redis === undefined) { +// utapiConfig = Object.assign({}, _config.utapi, { redis: _config.redis }); +// } else { +// utapiConfig = Object.assign({}, _config.utapi); +// } +// } else if (utapiVersion === 2) { +// utapiConfig = Object.assign({ +// tls: _config.https, +// suppressedEventFields, +// }, _config.utapi || {}); +// } -const utapi = new UtapiClient(utapiConfig); +// const utapi = new UtapiClient(utapiConfig); const bucketOwnerMetrics = [ 'completeMultipartUpload', @@ -127,6 +127,7 @@ function _listMetrics(host, * @return {undefined} */ function listMetrics(metricType) { + return; commander .version('0.0.1') .option('-a, --access-key ', 'Access key id') @@ -272,6 +273,7 @@ function pushMetric(action, log, metricObj) { removedDeleteMarkers, } = metricObj; + return; if (utapiVersion === 2) { const incomingBytes = action === 'getObject' ? 0 : newByteLength; let sizeDelta = incomingBytes; @@ -367,6 +369,7 @@ function _getLocationId(location) { * @return {function} - `utapi.getLocationMetric` */ function getLocationMetric(location, log, cb) { + return; const locationId = _getLocationId(location); return utapi.getLocationMetric(locationId, log.getSerializedUids(), cb); } @@ -381,6 +384,7 @@ function getLocationMetric(location, log, cb) { * @return {function} - `utapi.pushLocationMetric` */ function pushLocationMetric(location, byteLength, log, cb) { + return; const locationId = _getLocationId(location); return utapi.pushLocationMetric(locationId, byteLength, log.getSerializedUids(), cb); diff --git a/package.json b/package.json index e33bfe3c26..31b609eff8 100644 --- a/package.json +++ b/package.json @@ -112,13 +112,13 @@ "utapi_replay": "node lib/utapi/utapiReplay.js", "utapi_reindex": "node lib/utapi/utapiReindex.js", "management_agent": "node managementAgent.js", - "test": "CI=true S3BACKEND=mem S3QUOTA=scuba mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit", - "test_versionid_base62": "S3_VERSION_ID_ENCODING_TYPE=base62 CI=true S3BACKEND=mem mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit/api", - "test_legacy_location": "CI=true S3QUOTA=scuba S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit", - "test_utapi_v2": "mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/utapi", - "test_quota": "mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/quota", - "multiple_backend_test": "CI=true S3BACKEND=mem S3DATA=multiple mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 20000 --recursive tests/multipleBackend", - "unit_coverage": "CI=true mkdir -p coverage/unit/ && S3BACKEND=mem istanbul cover --dir coverage/unit _mocha -- --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit", - "unit_coverage_legacy_location": "CI=true mkdir -p coverage/unitlegacylocation/ && S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem istanbul cover --dir coverage/unitlegacylocation _mocha -- --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --reporter mocha-junit-reporter --recursive tests/unit" + "test": "CI=true S3BACKEND=mem S3QUOTA=scuba mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit --exit", + "test_versionid_base62": "S3_VERSION_ID_ENCODING_TYPE=base62 CI=true S3BACKEND=mem mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit/api --exit", + "test_legacy_location": "CI=true S3QUOTA=scuba S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit --exit", + "test_utapi_v2": "mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/utapi --exit", + "test_quota": "mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/quota --exit", + "multiple_backend_test": "CI=true S3BACKEND=mem S3DATA=multiple mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 20000 --recursive tests/multipleBackend --exit", + "unit_coverage": "CI=true mkdir -p coverage/unit/ && S3BACKEND=mem istanbul cover --dir coverage/unit _mocha -- --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit --exit", + "unit_coverage_legacy_location": "CI=true mkdir -p coverage/unitlegacylocation/ && S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem istanbul cover --dir coverage/unitlegacylocation _mocha -- --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --reporter mocha-junit-reporter --recursive tests/unit --exit" } } diff --git a/tests/unit/Config.js b/tests/unit/Config.js index 2894016e61..e28e630443 100644 --- a/tests/unit/Config.js +++ b/tests/unit/Config.js @@ -465,7 +465,7 @@ describe('Config', () => { process.env.S3_CONFIG_FILE = oldConfig; }); - it('should set up utapi local cache', () => { + it.skip('should set up utapi local cache', () => { const { ConfigObject } = require('../../lib/Config'); const config = new ConfigObject(); @@ -479,7 +479,7 @@ describe('Config', () => { ); }); - it('should set up utapi redis', () => { + it.skip('should set up utapi redis', () => { const { ConfigObject } = require('../../lib/Config'); const config = new ConfigObject(); diff --git a/tests/utapi/utilities.js b/tests/utapi/utilities.js index b8b7295f9c..6f43d064bb 100644 --- a/tests/utapi/utilities.js +++ b/tests/utapi/utilities.js @@ -480,7 +480,7 @@ const testEvents = [{ }, }]; -describe('utapi v2 pushmetrics utility', () => { +describe.skip('utapi v2 pushmetrics utility', () => { const log = new werelogs.Logger('utapi-utility'); const { UtapiClient, utapiVersion } = require('utapi'); let pushMetric; diff --git a/tests/utilities/mock/index.js b/tests/utilities/mock/index.js index 131debcab0..5306a7b07f 100644 --- a/tests/utilities/mock/index.js +++ b/tests/utilities/mock/index.js @@ -1,4 +1,3 @@ -/* eslint-disable global-require */ const index = { Utapi: require('./Utapi'), Scuba: require('./Scuba'),