WIP

app/Console/Commands/SetAllPublisherPublishedFileSizes.php

@@ -0,0 +1,55 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\IATI\Models\Activity\ActivityPublished;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
+
+class SetAllPublisherPublishedFileSizes extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'command:SetAllPublisherPublishedFileSizes';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Command description';
+
+    /**
+     * Execute the console command.
+     *
+     * @return void
+     */
+    public function handle(): void
+    {
+        ActivityPublished::chunk(50, function ($activityPublishedChunk) {
+            DB::beginTransaction();
+
+            try {
+                foreach ($activityPublishedChunk as $activityPublished) {
+                    $filename = $activityPublished->filename;
+                    $xmlString = awsGetFile("xml/mergedActivityXml/$filename");
+                    $fileSize = $xmlString ? calculateStringSizeInMb($xmlString) : 0;
+
+                    $activityPublished->filesize = $fileSize;
+                    $activityPublished->save();
+                    $this->info("Completed for org_id: $activityPublished->organization_id");
+                }
+
+                DB::commit();
+            } catch (\Exception $e) {
+                DB::rollBack();
+
+                $this->error('Error updating activity published file sizes: ' . $e->getMessage());
+            }
+        });
+
+    }
+}

app/Constants/DBTables.php

@@ -68,4 +68,9 @@ abstract class DBTables
      * For Validating Activity in IATI Registry.
      */
     public const VALIDATION_STATUS = 'validation_status';
+
+    /**
+     * For activity_published table.
+     */
+    public const ACTIVITY_PUBLISHED = 'activity_published';
 }

app/Constants/Enums.php

@@ -128,6 +128,7 @@ abstract class Enums
         'processing',
         'completed',
         'failed',
+        'max_merge_size_exception',
     ];
 
     /**
@@ -169,4 +170,6 @@ abstract class Enums
     public const MERGED_XML_BASE_PATH = 'xml/mergedActivityXml';
 
     public const ORG_XML_BASE_PATH = 'organizationXmlFiles';
+
+    public const MAX_MERGE_SIZE = '60';
 }

app/Exceptions/MaxMergeSizeExceededException.php

@@ -0,0 +1,18 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Exceptions;
+
+use Exception;
+
+/**
+ * class MaxMergeSizeExceededException.
+ */
+class MaxMergeSizeExceededException extends Exception
+{
+    public function __construct($message = 'Merged file max size exceeded', $code = 0, Exception $previous = null)
+    {
+        parent::__construct($message, $code, $previous);
+    }
+}

app/Helpers/general.php

@@ -1423,3 +1423,92 @@ function canAddMore($element): bool
         return Arr::get($element, 'add_more', false) || Arr::get($element, 'add_more_attributes', false);
     }
 }
+
+if (!function_exists('calculateStringSizeInMb')) {
+    /**
+     * @param string $stringVal
+     *
+     * @return float|int
+     */
+    function calculateStringSizeInMb(string $stringVal): float|int
+    {
+        return strlen($stringVal) / 1048576;
+    }
+}
+
+if (!function_exists('mapErrorLinesToChildren')) {
+    function mapErrorLinesToChildren($childXmlLineDetail, $originalErrorLineNumbers, $arrayWithErrorLineNumbersInsideTextMessage): array
+    {
+        $order = Arr::get($childXmlLineDetail, 'order');
+        $offset = Arr::get($childXmlLineDetail, 'beginsOn');
+
+        if ($order !== 0) {
+            $arrayWithErrorLineNumbersInsideTextMessage = json_encode($arrayWithErrorLineNumbersInsideTextMessage);
+
+            foreach ($originalErrorLineNumbers as $key => $lineNumber) {
+                $newLineNumber = $lineNumber - $offset;
+                $newLineNumber = str_contains($key, 'details') ? $newLineNumber + $order : $newLineNumber;
+                $newLineNumber = str_contains($key, 'lineNumber') ? $newLineNumber + 2 : $newLineNumber + 1;
+
+                $arrayWithErrorLineNumbersInsideTextMessage = str_replace("At line: $lineNumber", "At line: $newLineNumber", $arrayWithErrorLineNumbersInsideTextMessage);
+                $arrayWithErrorLineNumbersInsideTextMessage = str_replace("at line: $lineNumber", "at line: $newLineNumber", $arrayWithErrorLineNumbersInsideTextMessage);
+
+                $originalErrorLineNumbers[$key] = $newLineNumber;
+            }
+
+            $arrayWithErrorLineNumbersInsideTextMessage = json_decode($arrayWithErrorLineNumbersInsideTextMessage, true);
+        }
+
+        return [$originalErrorLineNumbers, $arrayWithErrorLineNumbersInsideTextMessage];
+    }
+}
+
+if (!function_exists('getItemsWhereKeyContains')) {
+    function getItemsWhereKeyContains(array $array, string $searchString): array
+    {
+        return Arr::where($array, function ($value, $key) use ($searchString) {
+            return str_contains($key, $searchString);
+        });
+    }
+}
+
+if (!function_exists('getLineNumbersOfEachActivity')) {
+    function getLineNumbersOfEachActivity($xmlDoc, array $uniqueIdentifiers, array $individualActivityXmlLength): array
+    {
+        $sumOfPreviousNodes = 0;
+        $result = [];
+        $xpath = new DOMXPath($xmlDoc);
+        $nodes = $xpath->query('//iati-activities/iati-activity/iati-identifier');
+
+        foreach ($nodes as $index => $node) {
+            $identifier = $node->textContent;
+
+            if (in_array($identifier, $uniqueIdentifiers)) {
+                if ($index == 0) {
+                    $lineNumber = 1;
+                    $sumOfPreviousNodes = $individualActivityXmlLength[$identifier] + 2; // this is correct
+                } else {
+                    $lineNumber = $sumOfPreviousNodes + $index;
+                    $sumOfPreviousNodes = $sumOfPreviousNodes + $individualActivityXmlLength[$identifier]; //55 -> 55+36
+                }
+
+                $result[$identifier] = [
+                    'order' => $index,
+                    'beginsOn' => $lineNumber,
+                    'individualLength' => $individualActivityXmlLength[$identifier],
+                ];
+            }
+        }
+
+        return $result;
+    }
+}
+
+if (!function_exists('filterErrorsByIdentifier')) {
+    function filterErrorsByIdentifier(array $errors, string $identifier): array
+    {
+        return array_values(array_filter($errors, function ($error) use ($identifier) {
+            return $error['identifier'] === $identifier;
+        }));
+    }
+}

app/IATI/Models/Activity/ActivityPublished.php

@@ -31,6 +31,7 @@ class ActivityPublished extends Model implements Auditable
             'filename',
             'published_to_registry',
             'organization_id',
+            'filesize',
             'created_at',
             'updated_at',
         ];

app/IATI/Repositories/Activity/ActivityRepository.php

@@ -84,7 +84,7 @@ public function getActivityForOrganization($organizationId, array $queryParams =
 
         $orderBy = 'updated_at';
         $direction = 'desc';
-        $limit = '10';
+        $limit = '100';
 
         if (array_key_exists('orderBy', $queryParams) && !empty($queryParams['orderBy'])) {
             $orderBy = $queryParams['orderBy'];

app/IATI/Repositories/Activity/ValidationStatusRepository.php

@@ -129,6 +129,12 @@ public function getActivitiesValidationStatus(array $activityIds): array
                     $allCompleted = false;
                 }
 
+                if ($validatorStatus->status === 'max_merge_size_exception') {
+                    $response['failed_count']++;
+                    $result[$validatorStatus->activity_id]['is_valid'] = false;
+                    $response['error_type'] = 'max_merge_size_exception';
+                }
+
                 if ($validatorStatus->status === 'failed') {
                     $response['failed_count']++;
                     $result[$validatorStatus->activity_id]['is_valid'] = false;

app/IATI/Services/Organization/OrganizationService.php

@@ -230,12 +230,13 @@ public function isPublisherStateActive(string $publisher_id): bool
             return false;
         }
 
-        $response = json_decode($res->getBody()->getContents(), false, 512, JSON_THROW_ON_ERROR);
-        $result = $response->result;
-
-        if (strcasecmp($result->state, 'active') === 0) {
-            return true;
-        }
+        return true;
+//        $response = json_decode($res->getBody()->getContents(), false, 512, JSON_THROW_ON_ERROR);
+//        $result = $response->result;
+//
+//        if (strcasecmp($result->state, 'active') === 0) {
+//            return true;
+//        }
 
         return false;
     }

app/IATI/Services/Workflow/ActivityWorkflowService.php

@@ -285,7 +285,6 @@ public function validateActivityOnIATIValidator($activity): string
      */
     public function getResponse($xmlData): string
     {
-        file_put_contents(storage_path('test.xml'), $xmlData);
         $client = new Client();
         $URI = env('IATI_VALIDATOR_ENDPOINT');
         $params['headers'] = ['Content-Type' => 'application/json', 'Ocp-Apim-Subscription-Key' => env('IATI_VALIDATOR_KEY')];
@@ -415,4 +414,12 @@ public function populateSectorIfMissing($activity): object
 
         return $activity->refresh();
     }
+
+    /**
+     * @throws GuzzleException
+     */
+    public function validateMultipleActivities(string $xmlData): string
+    {
+        return $this->getResponse($xmlData);
+    }
 }

app/IATI/Services/Workflow/BulkPublishingService.php

@@ -11,6 +11,7 @@
 use App\IATI\Services\Validator\ActivityValidatorResponseService;
 use App\IATI\Traits\IatiValidatorResponseTrait;
 use App\Jobs\RegistryValidatorJob;
+use App\Jobs\RegistryValidatorJobForMultipleActivities;
 use App\XlsImporter\Foundation\Factory\Validation;
 use GuzzleHttp\Exception\BadResponseException;
 use GuzzleHttp\Exception\GuzzleException;
@@ -138,7 +139,7 @@ public function getCompleteStatus($activity): string
     public function validateActivitiesOnIATI($activityIds): array
     {
         $user = Auth::user();
-        $activityTitle = [];
+        $activityTitles = [];
         $activities = $this->activityService->getActivitiesHavingIds($activityIds);
 
         /** @var $validationStatusRepository ValidationStatusRepository */
@@ -147,12 +148,24 @@ public function validateActivitiesOnIATI($activityIds): array
 
         foreach ($activities as $activity) {
             if ($activity && $activity->status === 'draft') {
-                $activityTitle[] = $activity->default_title_narrative;
+                $activityTitles[] = $activity->default_title_narrative;
+            }
+        }
+
+        foreach ($activities as $activity) {
+            if ($activity && $activity->status === 'draft' && count($activityTitles) === 1) {
                 RegistryValidatorJob::dispatch($activity, $user);
             }
         }
 
-        return $activityTitle;
+        if (count($activityTitles) > 1) {
+            $anActivity = $activities[0];
+            $organisation = $anActivity->organization;
+            $settings = $organisation->settings;
+            RegistryValidatorJobForMultipleActivities::dispatch($user, $activities, $organisation, $settings);
+        }
+
+        return $activityTitles;
     }
 
     /**

app/IATI/Traits/IatiValidatorResponseTrait.php

@@ -49,7 +49,6 @@ public function addElementOnIatiValidatorResponse($response, $activity): array
         $xmlString = $xml->asXML();
 
         if (!empty($errors)) {
-            logger()->info('Getting validation errors');
             foreach ($errors as $error) {
                 $updatedErrors[] = $this->getValidatorErrors($activity, $error, $xml, $xmlString);
             }

app/Jobs/RegistryValidatorJob.php

@@ -112,6 +112,7 @@ public function storeValidation($response): void
         $validatorService = app()->make(ActivityValidatorResponseService::class);
         $validationStatusRepository = app()->make(ValidationStatusRepository::class);
         $response = $this->addElementOnIatiValidatorResponse($response, $this->activity);
+
         $apiLogService->store(generateApiInfo('POST', env('IATI_VALIDATOR_ENDPOINT'), ['form_params' => json_encode($this->activity)], json_encode($response)));
 
         $recordResponse = [