Files
Warner-Vistex-Automater/app/Import/ImportRepository.php
T
Stefan Lazic a451060c1b Initial
Takes in a file, process it and output a new file with additional columns.
2026-07-10 15:45:02 +01:00

723 lines
26 KiB
PHP

<?php
declare(strict_types=1);
namespace App\Import;
use App\Database;
use PDO;
final class ImportRepository
{
public function __construct(private readonly PDO $pdo)
{
}
public static function fromDatabase(Database $database): self
{
return new self($database->pdo());
}
public function beginTransaction(): void
{
$this->pdo->beginTransaction();
}
public function commit(): void
{
$this->pdo->commit();
}
public function rollBack(): void
{
if ($this->pdo->inTransaction()) {
$this->pdo->rollBack();
}
}
public function clearAll(): void
{
$this->pdo->exec('TRUNCATE TABLE import_rows, import_batches RESTART IDENTITY CASCADE');
}
public function createBatch(string $sourceFilename, array $originalHeaders, array $calculatedHeaders, string $status = 'processing'): int
{
$statement = $this->pdo->prepare(
'INSERT INTO import_batches (source_filename, original_headers, calculated_headers, status) VALUES (:source_filename, CAST(:original_headers AS jsonb), CAST(:calculated_headers AS jsonb), :status) RETURNING id'
);
$statement->execute([
'source_filename' => $sourceFilename,
'original_headers' => json_encode(array_values($originalHeaders), JSON_THROW_ON_ERROR),
'calculated_headers' => json_encode(array_values($calculatedHeaders), JSON_THROW_ON_ERROR),
'status' => $status,
]);
return (int) $statement->fetchColumn();
}
public function updateBatchSummary(int $batchId, int $rowCount, int $warningsCount, string $status = 'processing'): void
{
$statement = $this->pdo->prepare('UPDATE import_batches SET row_count = :row_count, warnings_count = :warnings_count, status = :status WHERE id = :id');
$statement->execute([
'row_count' => $rowCount,
'warnings_count' => $warningsCount,
'status' => $status,
'id' => $batchId,
]);
}
public function insertRow(int $batchId, int $rowNumber, array $sourceRow, array $normalizedRow, array $calculatedRow, array $mergedRow): void
{
$statement = $this->pdo->prepare(
'INSERT INTO import_rows (batch_id, row_number, source_row, normalized_row, calculated_row, merged_row) VALUES (:batch_id, :row_number, CAST(:source_row AS jsonb), CAST(:normalized_row AS jsonb), CAST(:calculated_row AS jsonb), CAST(:merged_row AS jsonb))'
);
$statement->execute([
'batch_id' => $batchId,
'row_number' => $rowNumber,
'source_row' => json_encode($sourceRow, JSON_THROW_ON_ERROR),
'normalized_row' => json_encode($normalizedRow, JSON_THROW_ON_ERROR),
'calculated_row' => json_encode($calculatedRow, JSON_THROW_ON_ERROR),
'merged_row' => json_encode($mergedRow, JSON_THROW_ON_ERROR),
]);
}
public function getLatestBatch(): ?array
{
$statement = $this->pdo->query("SELECT * FROM import_batches WHERE status = 'complete' ORDER BY id DESC LIMIT 1");
$batch = $statement->fetch();
return $batch === false ? null : $this->decodeBatchRow($batch);
}
public function countRowsForBatch(int $batchId): int
{
$statement = $this->pdo->prepare('SELECT COUNT(*) FROM import_rows WHERE batch_id = :batch_id');
$statement->execute(['batch_id' => $batchId]);
return (int) $statement->fetchColumn();
}
public function getDistinctValuesForBatch(int $batchId, string $header, int $limit = 200): array
{
$statement = $this->pdo->prepare(
"SELECT DISTINCT NULLIF(TRIM(COALESCE(jsonb_extract_path_text(merged_row, :header), '')), '') AS value
FROM import_rows
WHERE batch_id = :batch_id
ORDER BY value ASC
LIMIT :limit"
);
$statement->bindValue('batch_id', $batchId, PDO::PARAM_INT);
$statement->bindValue('header', $header, PDO::PARAM_STR);
$statement->bindValue('limit', $limit, PDO::PARAM_INT);
$statement->execute();
$values = [];
while ($row = $statement->fetch()) {
$value = (string) ($row['value'] ?? '');
if ($value !== '') {
$values[] = $value;
}
}
return $values;
}
public function getDistinctValuesForBatchGrid(
int $batchId,
array $tableHeaders,
string $columnKey,
string $distinctSearch = '',
int $limit = 500
): array
{
if ($columnKey === 'row_number') {
return [];
}
$columnSql = $this->resolveGridColumnSql($tableHeaders, $columnKey);
if ($columnSql === null) {
return [];
}
$limit = max(1, min(500, $limit));
$distinctSearch = trim($distinctSearch);
$valueSql = 'COALESCE(' . $columnSql . ", '')";
$whereParts = ['batch_id = :batch_id'];
$parameters = [
'batch_id' => $batchId,
];
if ($distinctSearch !== '') {
$whereParts[] = $valueSql . ' ILIKE :distinct_search ESCAPE :distinct_search_esc';
$parameters['distinct_search'] = '%' . $this->escapeLikePattern($distinctSearch) . '%';
$parameters['distinct_search_esc'] = '\\';
}
$statement = $this->pdo->prepare(
'SELECT distinct_values.value
FROM (
SELECT DISTINCT ' . $valueSql . ' AS value
FROM import_rows
WHERE ' . implode(' AND ', $whereParts) . '
) AS distinct_values
ORDER BY LOWER(distinct_values.value) ASC, distinct_values.value ASC
LIMIT :limit'
);
foreach ($parameters as $name => $value) {
$statement->bindValue(':' . $name, $value, $name === 'batch_id' ? PDO::PARAM_INT : PDO::PARAM_STR);
}
$statement->bindValue(':limit', $limit, PDO::PARAM_INT);
$statement->execute();
$values = [];
while ($row = $statement->fetch()) {
$value = (string) ($row['value'] ?? '');
if (!in_array($value, $values, true)) {
$values[] = $value;
}
}
return $values;
}
public function paginateRows(int $batchId, int $limit, int $offset, string $sortDirection = 'asc'): array
{
return $this->paginateRowsFiltered($batchId, $limit, $offset, '', [], [], null, $sortDirection);
}
public function countRowsForBatchFiltered(
int $batchId,
string $query = '',
array $tableHeaders = [],
array $columnSearches = []
): int
{
[$whereClause, $parameters] = $this->buildDataTableClause($tableHeaders, $query, $columnSearches);
$statement = $this->pdo->prepare(
'SELECT COUNT(*)
FROM import_rows
WHERE batch_id = :batch_id' . $whereClause
);
$statement->bindValue(':batch_id', $batchId, PDO::PARAM_INT);
foreach ($parameters as $name => $value) {
$statement->bindValue(':' . $name, $value, PDO::PARAM_STR);
}
$statement->execute();
return (int) $statement->fetchColumn();
}
public function countRowsForBatchGrid(
int $batchId,
string $query = '',
array $tableHeaders = [],
array $filterModel = []
): int
{
[$whereClause, $parameters] = $this->buildGridWhereClause($tableHeaders, $query, $filterModel);
$statement = $this->pdo->prepare(
'SELECT COUNT(*)
FROM import_rows
WHERE batch_id = :batch_id' . $whereClause
);
$statement->bindValue(':batch_id', $batchId, PDO::PARAM_INT);
foreach ($parameters as $name => $value) {
$statement->bindValue(':' . $name, $value, PDO::PARAM_STR);
}
$statement->execute();
return (int) $statement->fetchColumn();
}
public function paginateRowsFiltered(
int $batchId,
int $limit,
int $offset,
string $query = '',
array $tableHeaders = [],
array $columnSearches = [],
?int $orderColumn = null,
string $sortDirection = 'asc'
): array
{
[$whereClause, $parameters] = $this->buildDataTableClause($tableHeaders, $query, $columnSearches);
[$orderClause, $orderParameters] = $this->buildOrderClause($tableHeaders, $orderColumn, $sortDirection);
$statement = $this->pdo->prepare(
'SELECT row_number, source_row, normalized_row, calculated_row, merged_row
FROM import_rows
WHERE batch_id = :batch_id' . $whereClause . '
' . $orderClause . '
LIMIT :limit OFFSET :offset'
);
$statement->bindValue(':batch_id', $batchId, PDO::PARAM_INT);
$statement->bindValue(':limit', $limit, PDO::PARAM_INT);
$statement->bindValue(':offset', $offset, PDO::PARAM_INT);
foreach ($parameters as $name => $value) {
$statement->bindValue(':' . $name, $value, PDO::PARAM_STR);
}
foreach ($orderParameters as $name => $value) {
$statement->bindValue(':' . $name, $value, PDO::PARAM_STR);
}
$statement->execute();
$rows = [];
while ($row = $statement->fetch()) {
$rows[] = $this->decodeRowPayload($row);
}
return $rows;
}
public function paginateRowsGrid(
int $batchId,
int $limit,
int $offset,
string $query = '',
array $tableHeaders = [],
array $filterModel = [],
array $sortModel = []
): array
{
[$whereClause, $parameters] = $this->buildGridWhereClause($tableHeaders, $query, $filterModel);
[$orderClause, $orderParameters] = $this->buildGridOrderClause($tableHeaders, $sortModel);
$statement = $this->pdo->prepare(
'SELECT row_number, merged_row
FROM import_rows
WHERE batch_id = :batch_id' . $whereClause . '
' . $orderClause . '
LIMIT :limit OFFSET :offset'
);
$statement->bindValue(':batch_id', $batchId, PDO::PARAM_INT);
$statement->bindValue(':limit', $limit, PDO::PARAM_INT);
$statement->bindValue(':offset', $offset, PDO::PARAM_INT);
foreach ($parameters as $name => $value) {
$statement->bindValue(':' . $name, $value, PDO::PARAM_STR);
}
foreach ($orderParameters as $name => $value) {
$statement->bindValue(':' . $name, $value, PDO::PARAM_STR);
}
$statement->execute();
$rows = [];
while ($row = $statement->fetch()) {
$rows[] = $this->decodeGridRowPayload($row);
}
return $rows;
}
public function getAllRowsCount(): int
{
return (int) $this->pdo->query('SELECT COUNT(*) FROM import_rows')->fetchColumn();
}
/**
* @return array<int, array<string, mixed>>
*/
public function exportRowsForBatch(int $batchId, int $limit, int $offset): array
{
$statement = $this->pdo->prepare(
'SELECT row_number, merged_row
FROM import_rows
WHERE batch_id = :batch_id
ORDER BY row_number ASC
LIMIT :limit OFFSET :offset'
);
$statement->bindValue(':batch_id', $batchId, PDO::PARAM_INT);
$statement->bindValue(':limit', $limit, PDO::PARAM_INT);
$statement->bindValue(':offset', $offset, PDO::PARAM_INT);
$statement->execute();
$rows = [];
while ($row = $statement->fetch()) {
$rows[] = $this->decodeGridRowPayload($row);
}
return $rows;
}
public function pruneExceptBatch(int $batchId): void
{
$statement = $this->pdo->prepare('DELETE FROM import_batches WHERE id <> :batch_id');
$statement->execute(['batch_id' => $batchId]);
}
private function decodeBatchRow(array $row): array
{
$row['original_headers'] = json_decode((string) $row['original_headers'], true, 512, JSON_THROW_ON_ERROR);
$row['calculated_headers'] = json_decode((string) $row['calculated_headers'], true, 512, JSON_THROW_ON_ERROR);
return $row;
}
private function decodeRowPayload(array $row): array
{
$row['source_row'] = json_decode((string) $row['source_row'], true, 512, JSON_THROW_ON_ERROR);
$row['normalized_row'] = json_decode((string) $row['normalized_row'], true, 512, JSON_THROW_ON_ERROR);
$row['calculated_row'] = json_decode((string) $row['calculated_row'], true, 512, JSON_THROW_ON_ERROR);
$row['merged_row'] = json_decode((string) $row['merged_row'], true, 512, JSON_THROW_ON_ERROR);
return $row;
}
private function decodeGridRowPayload(array $row): array
{
$mergedRow = json_decode((string) $row['merged_row'], true, 512, JSON_THROW_ON_ERROR);
if (!is_array($mergedRow)) {
$mergedRow = [];
}
return array_merge($mergedRow, [
'row_number' => (int) $row['row_number'],
]);
}
private function resolveGridColumnSql(array $tableHeaders, string $columnKey): ?string
{
if ($columnKey === 'row_number') {
return 'row_number';
}
$headerIndex = array_search($columnKey, $tableHeaders, true);
if ($headerIndex === false) {
return null;
}
return $this->columnSqlForHeader((string) $tableHeaders[(int) $headerIndex]);
}
/**
* @return array{0:string,1:array<string,string>}
*/
private function buildGridWhereClause(array $tableHeaders, string $globalSearch, array $filterModel): array
{
$conditions = [];
$parameters = [];
$globalSearch = trim($globalSearch);
if ($globalSearch !== '') {
$globalQueryRow = 'grid_global_query_row';
$globalQueryMerged = 'grid_global_query_merged';
$conditions[] = "(
CAST(row_number AS TEXT) ILIKE :{$globalQueryRow} ESCAPE :{$globalQueryRow}_esc
OR CAST(merged_row AS TEXT) ILIKE :{$globalQueryMerged} ESCAPE :{$globalQueryMerged}_esc
)";
$escapedGlobalSearch = '%' . $this->escapeLikePattern($globalSearch) . '%';
$parameters[$globalQueryRow] = $escapedGlobalSearch;
$parameters[$globalQueryRow . '_esc'] = '\\';
$parameters[$globalQueryMerged] = $escapedGlobalSearch;
$parameters[$globalQueryMerged . '_esc'] = '\\';
}
foreach ($filterModel as $columnId => $model) {
if (!is_array($model)) {
continue;
}
$columnKey = (string) $columnId;
$columnSql = $this->resolveGridColumnSql($tableHeaders, $columnKey);
if ($columnSql === null) {
continue;
}
$counter = 0;
$clause = $this->buildGridFilterClause($columnSql, $model, 'grid_' . preg_replace('/[^a-zA-Z0-9_]+/', '_', $columnKey), $parameters, $counter);
if ($clause !== '') {
$conditions[] = $clause;
}
}
if ($conditions === []) {
return ['', []];
}
return [
' AND (' . implode(' AND ', $conditions) . ')',
$parameters,
];
}
/**
* @return array{0:string,1:array<string,string>}
*/
private function buildGridOrderClause(array $tableHeaders, array $sortModel): array
{
$orderParts = [];
$parameters = [];
$hasRowNumberSort = false;
foreach ($sortModel as $index => $sortItem) {
if (!is_array($sortItem)) {
continue;
}
$columnKey = (string) ($sortItem['colId'] ?? '');
$sortDirection = strtolower((string) ($sortItem['sort'] ?? 'asc')) === 'desc' ? 'DESC' : 'ASC';
$columnSql = $this->resolveGridColumnSql($tableHeaders, $columnKey);
if ($columnSql === null) {
continue;
}
if ($columnKey === 'row_number') {
$orderParts[] = 'row_number ' . $sortDirection;
$hasRowNumberSort = true;
continue;
}
$orderParts[] = 'LOWER(COALESCE(' . $columnSql . ", '')) " . $sortDirection;
}
if ($orderParts === []) {
return ['ORDER BY row_number ASC', []];
}
if (!$hasRowNumberSort) {
$orderParts[] = 'row_number ASC';
}
return ['ORDER BY ' . implode(', ', $orderParts), $parameters];
}
/**
* @return array<string,string>
*/
private function buildGridFilterClause(string $columnSql, array $model, string $paramPrefix, array &$parameters, int &$counter): string
{
if (isset($model['operator'], $model['condition1'], $model['condition2']) && is_array($model['condition1']) && is_array($model['condition2'])) {
$left = $this->buildGridFilterClause($columnSql, $model['condition1'], $paramPrefix . '_c1', $parameters, $counter);
$right = $this->buildGridFilterClause($columnSql, $model['condition2'], $paramPrefix . '_c2', $parameters, $counter);
$operator = strtoupper((string) $model['operator']) === 'OR' ? 'OR' : 'AND';
return '(' . $left . ' ' . $operator . ' ' . $right . ')';
}
$filterType = strtolower((string) ($model['filterType'] ?? 'text'));
if ($filterType === 'set') {
return $this->buildGridSetFilterClause($columnSql, $model, $paramPrefix, $parameters, $counter);
}
if ($filterType === 'number') {
return $this->buildGridNumberFilterClause($columnSql, $model, $paramPrefix, $parameters, $counter);
}
return $this->buildGridTextFilterClause($columnSql, $model, $paramPrefix, $parameters, $counter);
}
/**
* @return array<string,string>
*/
private function buildGridTextFilterClause(string $columnSql, array $model, string $paramPrefix, array &$parameters, int &$counter): string
{
$type = strtolower((string) ($model['type'] ?? 'contains'));
$filter = trim((string) ($model['filter'] ?? ''));
$paramName = $paramPrefix . '_' . $counter++;
$escapedFilter = $this->escapeLikePattern($filter);
return match ($type) {
'blank' => 'COALESCE(TRIM(' . $columnSql . "), '') = ''",
'notblank' => 'COALESCE(TRIM(' . $columnSql . "), '') <> ''",
'equals' => $this->addTextFilterParameter($columnSql, $paramName, $escapedFilter, $parameters, true),
'notequal' => $this->addTextFilterParameter($columnSql, $paramName, $escapedFilter, $parameters, false),
'startswith' => $this->addTextFilterParameter($columnSql, $paramName, $escapedFilter, $parameters, true, false, true),
'endswith' => $this->addTextFilterParameter($columnSql, $paramName, $escapedFilter, $parameters, true, true, false),
'notcontains' => $this->addTextFilterParameter($columnSql, $paramName, $escapedFilter, $parameters, false, true, true),
default => $this->addTextFilterParameter($columnSql, $paramName, $escapedFilter, $parameters, true, true, true),
};
}
/**
* @return array<string,string>
*/
private function buildGridNumberFilterClause(string $columnSql, array $model, string $paramPrefix, array &$parameters, int &$counter): string
{
$type = strtolower((string) ($model['type'] ?? 'equals'));
$filter = $model['filter'] ?? null;
$filterTo = $model['filterTo'] ?? null;
$paramName = $paramPrefix . '_' . $counter++;
$from = is_numeric($filter) ? (string) $filter : '';
$to = is_numeric($filterTo) ? (string) $filterTo : '';
return match ($type) {
'blank' => '1 = 0',
'notblank' => '1 = 1',
'notequal' => $this->addNumericFilterParameter($columnSql, $paramName, $from, $parameters, '<>'),
'lessthan' => $this->addNumericFilterParameter($columnSql, $paramName, $from, $parameters, '<'),
'lessthanorequal' => $this->addNumericFilterParameter($columnSql, $paramName, $from, $parameters, '<='),
'greaterthan' => $this->addNumericFilterParameter($columnSql, $paramName, $from, $parameters, '>'),
'greaterthanorequal' => $this->addNumericFilterParameter($columnSql, $paramName, $from, $parameters, '>='),
'inrange' => $this->addNumericRangeFilterParameter($columnSql, $paramName, $from, $to, $parameters),
default => $this->addNumericFilterParameter($columnSql, $paramName, $from, $parameters, '='),
};
}
/**
* @return array<string,string>
*/
private function buildGridSetFilterClause(string $columnSql, array $model, string $paramPrefix, array &$parameters, int &$counter): string
{
$values = $model['values'] ?? [];
if (!is_array($values) || $values === []) {
return '1 = 1';
}
$placeholders = [];
foreach ($values as $value) {
if (!is_scalar($value) && $value !== null) {
continue;
}
$paramName = $paramPrefix . '_' . $counter++;
$parameters[$paramName] = (string) $value;
$placeholders[] = ':' . $paramName;
}
if ($placeholders === []) {
return '1 = 1';
}
return $columnSql . ' IN (' . implode(', ', $placeholders) . ')';
}
private function addTextFilterParameter(
string $columnSql,
string $paramName,
string $escapedFilter,
array &$parameters,
bool $positive = true,
bool $prependWildcard = false,
bool $appendWildcard = false
): string
{
if ($escapedFilter === '') {
return '1 = 1';
}
$parameters[$paramName] = ($prependWildcard ? '%' : '') . $escapedFilter . ($appendWildcard ? '%' : '');
$parameters[$paramName . '_esc'] = '\\';
$clause = $columnSql . " ILIKE :{$paramName} ESCAPE :{$paramName}_esc";
return $positive ? $clause : 'NOT (' . $clause . ')';
}
private function addNumericFilterParameter(string $columnSql, string $paramName, string $value, array &$parameters, string $operator): string
{
if ($value === '') {
return '1 = 1';
}
$parameters[$paramName] = $value;
return $columnSql . " {$operator} :{$paramName}";
}
private function addNumericRangeFilterParameter(string $columnSql, string $paramName, string $from, string $to, array &$parameters): string
{
if ($from === '' || $to === '') {
return '1 = 1';
}
$fromName = $paramName . '_from';
$toName = $paramName . '_to';
$parameters[$fromName] = $from;
$parameters[$toName] = $to;
return $columnSql . " BETWEEN :{$fromName} AND :{$toName}";
}
private function columnSqlForHeader(string $header): string
{
return 'COALESCE(merged_row ->> ' . $this->pdo->quote($header) . ", '')";
}
private function escapeLikePattern(string $value): string
{
return str_replace(['\\', '%', '_'], ['\\\\', '\\%', '\\_'], $value);
}
private function isAssocArray(array $value): bool
{
return array_keys($value) !== range(0, count($value) - 1);
}
/**
* @return array{0:string,1:array<string,string>}
*/
private function buildDataTableClause(array $tableHeaders, string $globalSearch, array $columnSearches): array
{
$conditions = [];
$parameters = [];
$globalSearch = trim($globalSearch);
if ($globalSearch !== '') {
$conditions[] = "(
CAST(row_number AS TEXT) ILIKE :global_query ESCAPE :global_query_esc
OR CAST(merged_row AS TEXT) ILIKE :global_query ESCAPE :global_query_esc
)";
$parameters['global_query'] = '%' . str_replace(['\\', '%', '_'], ['\\\\', '\\%', '\\_'], $globalSearch) . '%';
$parameters['global_query_esc'] = '\\';
}
foreach ($columnSearches as $index => $search) {
$search = trim((string) $search);
if ($search === '') {
continue;
}
$parameterName = 'column_query_' . (int) $index;
if ((int) $index === 0) {
$conditions[] = "CAST(row_number AS TEXT) = :{$parameterName}";
$parameters[$parameterName] = $search;
continue;
}
$headerIndex = (int) $index - 1;
if (!isset($tableHeaders[$headerIndex])) {
continue;
}
$headerParameter = 'column_header_' . (int) $index;
$conditions[] = "COALESCE(jsonb_extract_path_text(merged_row, :{$headerParameter}), '') = :{$parameterName}";
$parameters[$headerParameter] = (string) $tableHeaders[$headerIndex];
$parameters[$parameterName] = $search;
}
if ($conditions === []) {
return ['', []];
}
return [
' AND (' . implode(' AND ', $conditions) . ')',
$parameters,
];
}
/**
* @return array{0:string,1:array<string,string>}
*/
private function buildOrderClause(array $tableHeaders, ?int $orderColumn, string $sortDirection): array
{
$sortDirection = strtoupper($sortDirection) === 'DESC' ? 'DESC' : 'ASC';
if ($orderColumn === null || $orderColumn < 0) {
return ['ORDER BY row_number DESC', []];
}
if ($orderColumn === 0) {
return ['ORDER BY row_number ' . $sortDirection, []];
}
$headerIndex = $orderColumn - 1;
if (!isset($tableHeaders[$headerIndex])) {
return ['ORDER BY row_number DESC', []];
}
return [
'ORDER BY LOWER(COALESCE(jsonb_extract_path_text(merged_row, :order_header), \'\')) ' . $sortDirection . ', row_number ASC',
[
'order_header' => (string) $tableHeaders[$headerIndex],
],
];
}
}