diff --git a/modules/backend/behaviors/ImportExportController.php b/modules/backend/behaviors/ImportExportController.php index 1663fadf0..dffa2ead5 100644 --- a/modules/backend/behaviors/ImportExportController.php +++ b/modules/backend/behaviors/ImportExportController.php @@ -7,6 +7,7 @@ use Response; use Backend; use BackendAuth; use Backend\Classes\ControllerBehavior; +use Backend\Classes\StreamFilterTranscode; use League\Csv\Reader as CsvReader; use League\Csv\Writer as CsvWriter; use ApplicationException; @@ -174,15 +175,22 @@ class ImportExportController extends ControllerBehavior try { $model = $this->importGetModel(); $matches = post('column_match', []); + $importOptions = [ + 'sessionKey' => $this->importUploadFormWidget->getSessionKey() + ]; if ($optionData = post('ImportOptions')) { $model->fill($optionData); } - $model->import($matches, [ - 'sessionKey' => $this->importUploadFormWidget->getSessionKey(), - 'firstRowTitles' => post('first_row_titles', false) - ]); + if (post('format_preset') == 'custom') { + $importOptions['delimiter'] = post('format_delimiter'); + $importOptions['enclosure'] = post('format_enclosure'); + $importOptions['escape'] = post('format_escape'); + $importOptions['encoding'] = post('format_encoding'); + } + + $model->import($matches, $importOptions); $this->vars['importResults'] = $model->getResultStats(); $this->vars['returnUrl'] = $this->getRedirectUrlForType('import'); @@ -218,7 +226,7 @@ class ImportExportController extends ControllerBehavior } $path = $this->getImportFilePath(); - $reader = CsvReader::createFromPath($path); + $reader = $this->createCsvReader($path); if (post('first_row_titles')) { $reader->setOffset(1); @@ -293,7 +301,7 @@ class ImportExportController extends ControllerBehavior return null; } - $reader = CsvReader::createFromPath($path); + $reader = $this->createCsvReader($path); $firstRow = $reader->fetchOne(0); if (!post('first_row_titles')) { @@ -302,6 +310,11 @@ class ImportExportController extends ControllerBehavior }); } + // Prevents unfriendly error to be thrown due to bad encoding at response time + if (false === json_encode($firstRow)) { + throw new ApplicationException(Lang::get('backend::lang.import_export.encoding_not_supported_error')); + } + return $firstRow; } @@ -691,4 +704,50 @@ class ImportExportController extends ControllerBehavior return $this->controller->actionUrl($type); } + + + /** + * Create a new CSV reader with options selected by the user + * @param string $path + * + * @return CsvReader + */ + protected function createCsvReader($path) + { + $reader = CsvReader::createFromPath($path); + + if (post('format_preset') == 'custom') { + $options = [ + 'delimiter' => post('format_delimiter'), + 'enclosure' => post('format_enclosure'), + 'escape' => post('format_escape'), + 'encoding' => post('format_encoding') + ]; + + if ($options['delimiter'] !== null) { + $reader->setDelimiter($options['delimiter']); + } + + if ($options['enclosure'] !== null) { + $reader->setEnclosure($options['enclosure']); + } + + if ($options['escape'] !== null) { + $reader->setEscape($options['escape']); + } + + if ($options['encoding'] !== null) { + if ($reader->isActiveStreamFilter()) { + $reader->appendStreamFilter(sprintf( + '%s%s:%s', + StreamFilterTranscode::FILTER_NAME, + strtolower($options['encoding']), + 'utf-8' + )); + } + } + } + + return $reader; + } } \ No newline at end of file diff --git a/modules/backend/behaviors/importexportcontroller/partials/fields_import.yaml b/modules/backend/behaviors/importexportcontroller/partials/fields_import.yaml index da4dbcf02..7c2397304 100644 --- a/modules/backend/behaviors/importexportcontroller/partials/fields_import.yaml +++ b/modules/backend/behaviors/importexportcontroller/partials/fields_import.yaml @@ -15,6 +15,52 @@ fields: fileTypes: csv useCaption: false + format_preset: + label: backend::lang.import_export.file_format + type: dropdown + default: standard + options: + standard: backend::lang.import_export.standard_format + custom: backend::lang.import_export.custom_format + span: right + + format_delimiter: + label: backend::lang.import_export.delimiter_char + default: ',' + span: left + trigger: + action: show + condition: value[custom] + field: format_preset + + format_enclosure: + label: backend::lang.import_export.enclosure_char + span: auto + default: '"' + trigger: + action: show + condition: value[custom] + field: format_preset + + format_escape: + label: backend::lang.import_export.escape_char + span: auto + default: '\' + trigger: + action: show + condition: value[custom] + field: format_preset + + format_encoding: + label: backend::lang.import_export.encoding_format + span: auto + default: UTF-8 + type: dropdown + trigger: + action: show + condition: value[custom] + field: format_preset + first_row_titles: label: backend::lang.import_export.first_row_contains_titles comment: backend::lang.import_export.first_row_contains_titles_desc @@ -34,14 +80,14 @@ fields: label: backend::lang.import_export.file_columns type: partial path: ~/modules/backend/behaviors/importexportcontroller/partials/_import_file_columns.htm - dependsOn: [import_file, first_row_titles] + dependsOn: [import_file, first_row_titles, format_delimiter, format_enclosure, format_escape, format_encoding] span: left import_db_columns: label: backend::lang.import_export.database_fields type: partial path: ~/modules/backend/behaviors/importexportcontroller/partials/_import_db_columns.htm - dependsOn: [import_file, first_row_titles] + dependsOn: [import_file, first_row_titles, format_delimiter, format_enclosure, format_escape, format_encoding] span: right step3_section: diff --git a/modules/backend/classes/StreamFilterTranscode.php b/modules/backend/classes/StreamFilterTranscode.php new file mode 100644 index 000000000..14b852b51 --- /dev/null +++ b/modules/backend/classes/StreamFilterTranscode.php @@ -0,0 +1,71 @@ +filtername, self::FILTER_NAME) !== 0) { + return false; + } + + $params = substr($this->filtername, strlen(self::FILTER_NAME)); + if ( ! preg_match('/^([-\w]+)(:([-\w]+))?$/', $params, $matches)) { + return false; + } + + if (isset( $matches[1] )) { + $this->encoding_from = $matches[1]; + } + + $this->encoding_to = mb_internal_encoding(); + if (isset( $matches[3] )) { + $this->encoding_to = $matches[3]; + } + + $this->params['locale'] = setlocale(LC_CTYPE, '0'); + if (stripos($this->params['locale'], 'UTF-8') === false) { + setlocale(LC_CTYPE, 'en_US.UTF-8'); + } + + return true; + } + + + public function onClose() + { + setlocale(LC_CTYPE, $this->params['locale']); + } + + + public function filter($in, $out, &$consumed, $closing) + { + while ($res = stream_bucket_make_writeable($in)) { + $res->data = @mb_convert_encoding($res->data, $this->encoding_to, $this->encoding_from); + $consumed += $res->datalen; + stream_bucket_append($out, $res); + } + + return PSFS_PASS_ON; + } +} \ No newline at end of file diff --git a/modules/backend/lang/en/lang.php b/modules/backend/lang/en/lang.php index 8b0f55d94..9a2bfc266 100644 --- a/modules/backend/lang/en/lang.php +++ b/modules/backend/lang/en/lang.php @@ -374,5 +374,27 @@ return [ 'column_preview' => 'Column preview', 'file_not_found_error' => 'File not found', 'empty_error' => 'There was no data supplied to export', + 'encoding_not_supported_error' => 'Source file encoding is not recognized. Please select the custom file format option with the proper encoding to import your file.', + 'encoding_format' => 'File encoding', + 'encodings' => [ + 'utf-8' => 'UTF-8', + 'us-ascii' => 'US-ASCII', + 'iso-8859-1' => 'ISO-8859-1 (Latin-1, Western European)', + 'iso-8859-2' => 'ISO-8859-2 (Latin-2, Central European)', + 'iso-8859-3' => 'ISO-8859-3 (Latin-3, South European)', + 'iso-8859-4' => 'ISO-8859-4 (Latin-4, North European)', + 'iso-8859-5' => 'ISO-8859-5 (Latin, Cyrillic)', + 'iso-8859-6' => 'ISO-8859-6 (Latin, Arabic)', + 'iso-8859-7' => 'ISO-8859-7 (Latin, Greek)', + 'iso-8859-8' => 'ISO-8859-8 (Latin, Hebrew)', + 'iso-8859-0' => 'ISO-8859-9 (Latin-5, Turkish)', + 'iso-8859-10' => 'ISO-8859-10 (Latin-6, Nordic)', + 'iso-8859-11' => 'ISO-8859-11 (Latin, Thai)', + 'iso-8859-13' => 'ISO-8859-13 (Latin-7, Baltic Rim)', + 'iso-8859-14' => 'ISO-8859-14 (Latin-8, Celtic)', + 'iso-8859-15' => 'ISO-8859-15 (Latin-9, Western European revision with euro sign)', + 'Windows-1251' => 'Windows-1251 (CP1251)', + 'Windows-1252' => 'Windows-1252 (CP1252)' + ], ], ]; diff --git a/modules/backend/lang/fr/lang.php b/modules/backend/lang/fr/lang.php index 94b3c770b..dd24e4856 100644 --- a/modules/backend/lang/fr/lang.php +++ b/modules/backend/lang/fr/lang.php @@ -358,5 +358,27 @@ return [ 'column_preview' => 'Prévisualisation des colonnes', 'file_not_found_error' => 'Fichier non trouvé', 'empty_error' => 'Il n‘y a aucune donnée à exporter', + 'encoding_not_supported_error' => 'L’encodage de votre fichier source n’est pas reconnu. Veuillez sélectionner le format d’import personnalisé avec l’encodage adapté pour importer votre fichier.', + 'encoding_format' => 'Encodage du fichier', + 'encodings' => [ + 'utf-8' => 'UTF-8', + 'us-ascii' => 'US-ASCII', + 'iso-8859-1' => 'ISO-8859-1 (Latin-1, européen occidental)', + 'iso-8859-2' => 'ISO-8859-2 (Latin-2, européen central)', + 'iso-8859-3' => 'ISO-8859-3 (Latin-3, européen du Sud)', + 'iso-8859-4' => 'ISO-8859-4 (Latin-4, européen du Nord)', + 'iso-8859-5' => 'ISO-8859-5 (Latin, cyrillique)', + 'iso-8859-6' => 'ISO-8859-6 (Latin, arabe)', + 'iso-8859-7' => 'ISO-8859-7 (Latin, grec)', + 'iso-8859-8' => 'ISO-8859-8 (Latin, hébreu)', + 'iso-8859-0' => 'ISO-8859-9 (Latin-5, turc)', + 'iso-8859-10' => 'ISO-8859-10 (Latin-6, nordique)', + 'iso-8859-11' => 'ISO-8859-11 (Latin, thaï)', + 'iso-8859-13' => 'ISO-8859-13 (Latin-7, balte)', + 'iso-8859-14' => 'ISO-8859-14 (Latin-8, celtique)', + 'iso-8859-15' => 'ISO-8859-15 (Latin-9, européen occidental révisé avec le signe euro)', + 'Windows-1251' => 'Windows-1251 (CP1251)', + 'Windows-1252' => 'Windows-1252 (CP1252)' + ], ], ]; diff --git a/modules/backend/models/ImportModel.php b/modules/backend/models/ImportModel.php index d34112f8f..c8668013e 100644 --- a/modules/backend/models/ImportModel.php +++ b/modules/backend/models/ImportModel.php @@ -87,10 +87,21 @@ abstract class ImportModel extends Model */ protected function processImportData($filePath, $matches, $options) { - extract(array_merge([ - 'firstRowTitles' => true - ], $options)); + /* + * Parse options + */ + $defaultOptions = [ + 'firstRowTitles' => true, + 'delimiter' => null, + 'enclosure' => null, + 'escape' => null + ]; + $options = array_merge($defaultOptions, $options); + + /* + * Read CSV + */ $reader = CsvReader::createFromPath($filePath, 'r'); // Filter out empty rows @@ -98,7 +109,19 @@ abstract class ImportModel extends Model return count($row) > 1 || reset($row) !== null; }); - if ($firstRowTitles) { + if ($options['delimiter'] !== null) { + $reader->setDelimiter($options['delimiter']); + } + + if ($options['enclosure'] !== null) { + $reader->setEnclosure($options['enclosure']); + } + + if ($options['escape'] !== null) { + $reader->setEscape($options['escape']); + } + + if ($options['firstRowTitles']) { $reader->setOffset(1); } @@ -165,6 +188,16 @@ abstract class ImportModel extends Model return $file->getLocalPath(); } + + /** + * Returns all available encodings values from the localization config + * @return array + */ + public function getFormatEncodingOptions() + { + return \Lang::get('backend::lang.import_export.encodings'); + } + // // Result logging //