Import options to configure the CSV reader + patch to prevent unfriendly error to be thrown due to bad import source file encoding (on AJAX request)

modules/backend/behaviors/ImportExportController.php

@@ -7,6 +7,7 @@ use Response;
 use Backend;
 use BackendAuth;
 use Backend\Classes\ControllerBehavior;
+use Backend\Classes\StreamFilterTranscode;
 use League\Csv\Reader as CsvReader;
 use League\Csv\Writer as CsvWriter;
 use ApplicationException;
@@ -174,15 +175,22 @@ class ImportExportController extends ControllerBehavior
         try {
             $model = $this->importGetModel();
             $matches = post('column_match', []);
+            $importOptions = [
+                'sessionKey' => $this->importUploadFormWidget->getSessionKey()
+            ];
 
             if ($optionData = post('ImportOptions')) {
                 $model->fill($optionData);
             }
 
-            $model->import($matches, [
+            if (post('format_preset') == 'custom') {
-                'sessionKey' => $this->importUploadFormWidget->getSessionKey(),
+                $importOptions['delimiter'] = post('format_delimiter');
-                'firstRowTitles' => post('first_row_titles', false)
+                $importOptions['enclosure'] = post('format_enclosure');
-            ]);
+                $importOptions['escape'] = post('format_escape');
+                $importOptions['encoding'] = post('format_encoding');
+            }
+
+            $model->import($matches, $importOptions);
 
             $this->vars['importResults'] = $model->getResultStats();
             $this->vars['returnUrl'] = $this->getRedirectUrlForType('import');
@@ -218,7 +226,7 @@ class ImportExportController extends ControllerBehavior
         }
 
         $path = $this->getImportFilePath();
-        $reader = CsvReader::createFromPath($path);
+        $reader   = $this->createCsvReader($path);
 
         if (post('first_row_titles')) {
             $reader->setOffset(1);
@@ -293,7 +301,7 @@ class ImportExportController extends ControllerBehavior
             return null;
         }
 
-        $reader = CsvReader::createFromPath($path);
+        $reader   = $this->createCsvReader($path);
         $firstRow = $reader->fetchOne(0);
 
         if (!post('first_row_titles')) {
@@ -302,6 +310,11 @@ class ImportExportController extends ControllerBehavior
             });
         }
 
+        // Prevents unfriendly error to be thrown due to bad encoding at response time
+        if (false === json_encode($firstRow)) {
+            throw new ApplicationException(Lang::get('backend::lang.import_export.encoding_not_supported_error'));
+        }
+
         return $firstRow;
     }
 
@@ -691,4 +704,50 @@ class ImportExportController extends ControllerBehavior
 
         return $this->controller->actionUrl($type);
     }
+
+
+    /**
+     * Create a new CSV reader with options selected by the user
+     * @param string $path
+     *
+     * @return CsvReader
+     */
+    protected function createCsvReader($path)
+    {
+        $reader = CsvReader::createFromPath($path);
+
+        if (post('format_preset') == 'custom') {
+            $options = [
+                'delimiter' => post('format_delimiter'),
+                'enclosure' => post('format_enclosure'),
+                'escape' => post('format_escape'),
+                'encoding' => post('format_encoding')
+            ];
+
+            if ($options['delimiter'] !== null) {
+                $reader->setDelimiter($options['delimiter']);
+            }
+
+            if ($options['enclosure'] !== null) {
+                $reader->setEnclosure($options['enclosure']);
+            }
+
+            if ($options['escape'] !== null) {
+                $reader->setEscape($options['escape']);
+            }
+
+            if ($options['encoding'] !== null) {
+                if ($reader->isActiveStreamFilter()) {
+                    $reader->appendStreamFilter(sprintf(
+                        '%s%s:%s',
+                        StreamFilterTranscode::FILTER_NAME,
+                        strtolower($options['encoding']),
+                        'utf-8'
+                    ));
+                }
+            }
+        }
+
+        return $reader;
+    }
 }

modules/backend/behaviors/importexportcontroller/partials/fields_import.yaml

@@ -15,6 +15,52 @@ fields:
         fileTypes: csv
         useCaption: false
 
+    format_preset:
+        label: backend::lang.import_export.file_format
+        type: dropdown
+        default: standard
+        options:
+            standard: backend::lang.import_export.standard_format
+            custom: backend::lang.import_export.custom_format
+        span: right
+
+    format_delimiter:
+        label: backend::lang.import_export.delimiter_char
+        default: ','
+        span: left
+        trigger:
+            action: show
+            condition: value[custom]
+            field: format_preset
+
+    format_enclosure:
+        label: backend::lang.import_export.enclosure_char
+        span: auto
+        default: '"'
+        trigger:
+            action: show
+            condition: value[custom]
+            field: format_preset
+
+    format_escape:
+        label: backend::lang.import_export.escape_char
+        span: auto
+        default: '\'
+        trigger:
+            action: show
+            condition: value[custom]
+            field: format_preset
+
+    format_encoding:
+        label: backend::lang.import_export.encoding_format
+        span: auto
+        default: UTF-8
+        type: dropdown
+        trigger:
+            action: show
+            condition: value[custom]
+            field: format_preset
+
     first_row_titles:
         label: backend::lang.import_export.first_row_contains_titles
         comment: backend::lang.import_export.first_row_contains_titles_desc
@@ -34,14 +80,14 @@ fields:
         label: backend::lang.import_export.file_columns
         type: partial
         path: ~/modules/backend/behaviors/importexportcontroller/partials/_import_file_columns.htm
-        dependsOn: [import_file, first_row_titles]
+        dependsOn: [import_file, first_row_titles, format_delimiter, format_enclosure, format_escape, format_encoding]
         span: left
 
     import_db_columns:
         label: backend::lang.import_export.database_fields
         type: partial
         path: ~/modules/backend/behaviors/importexportcontroller/partials/_import_db_columns.htm
-        dependsOn: [import_file, first_row_titles]
+        dependsOn: [import_file, first_row_titles, format_delimiter, format_enclosure, format_escape, format_encoding]
         span: right
 
     step3_section:

modules/backend/classes/StreamFilterTranscode.php

@@ -0,0 +1,71 @@
+<?php namespace Backend\Classes;
+
+use php_user_filter;
+
+// Register the class for it to be usable by the CSV Lib
+stream_filter_register(StreamFilterTranscode::FILTER_NAME . "*", StreamFilterTranscode::class);
+
+/**
+ * A universal transcode stream filter.
+ * Used by the backend import model to convert source file from one encoding to another.
+ * The system must support both source and target encoding encoding.
+ *
+ * @credits https://github.com/thephpleague/csv/blob/master/examples/lib/FilterTranscode.php
+ * @package october\backend
+ */
+class StreamFilterTranscode extends php_user_filter
+{
+
+    const FILTER_NAME = 'convert.transcode.';
+
+    private $encoding_from = 'auto';
+
+    private $encoding_to;
+
+
+    public function onCreate()
+    {
+        if (strpos($this->filtername, self::FILTER_NAME) !== 0) {
+            return false;
+        }
+
+        $params = substr($this->filtername, strlen(self::FILTER_NAME));
+        if ( ! preg_match('/^([-\w]+)(:([-\w]+))?$/', $params, $matches)) {
+            return false;
+        }
+
+        if (isset( $matches[1] )) {
+            $this->encoding_from = $matches[1];
+        }
+
+        $this->encoding_to = mb_internal_encoding();
+        if (isset( $matches[3] )) {
+            $this->encoding_to = $matches[3];
+        }
+
+        $this->params['locale'] = setlocale(LC_CTYPE, '0');
+        if (stripos($this->params['locale'], 'UTF-8') === false) {
+            setlocale(LC_CTYPE, 'en_US.UTF-8');
+        }
+
+        return true;
+    }
+
+
+    public function onClose()
+    {
+        setlocale(LC_CTYPE, $this->params['locale']);
+    }
+
+
+    public function filter($in, $out, &$consumed, $closing)
+    {
+        while ($res = stream_bucket_make_writeable($in)) {
+            $res->data = @mb_convert_encoding($res->data, $this->encoding_to, $this->encoding_from);
+            $consumed += $res->datalen;
+            stream_bucket_append($out, $res);
+        }
+
+        return PSFS_PASS_ON;
+    }
+}

modules/backend/lang/en/lang.php

@@ -374,5 +374,27 @@ return [
         'column_preview' => 'Column preview',
         'file_not_found_error' => 'File not found',
         'empty_error' => 'There was no data supplied to export',
+        'encoding_not_supported_error' => 'Source file encoding is not recognized. Please select the custom file format option with the proper encoding to import your file.',
+        'encoding_format' => 'File encoding',
+        'encodings' => [
+            'utf-8' => 'UTF-8',
+            'us-ascii' => 'US-ASCII',
+            'iso-8859-1' => 'ISO-8859-1 (Latin-1, Western European)',
+            'iso-8859-2' => 'ISO-8859-2 (Latin-2, Central European)',
+            'iso-8859-3' => 'ISO-8859-3 (Latin-3, South European)',
+            'iso-8859-4' => 'ISO-8859-4 (Latin-4, North European)',
+            'iso-8859-5' => 'ISO-8859-5 (Latin, Cyrillic)',
+            'iso-8859-6' => 'ISO-8859-6 (Latin, Arabic)',
+            'iso-8859-7' => 'ISO-8859-7 (Latin, Greek)',
+            'iso-8859-8' => 'ISO-8859-8 (Latin, Hebrew)',
+            'iso-8859-0' => 'ISO-8859-9 (Latin-5, Turkish)',
+            'iso-8859-10' => 'ISO-8859-10 (Latin-6, Nordic)',
+            'iso-8859-11' => 'ISO-8859-11 (Latin, Thai)',
+            'iso-8859-13' => 'ISO-8859-13 (Latin-7, Baltic Rim)',
+            'iso-8859-14' => 'ISO-8859-14 (Latin-8, Celtic)',
+            'iso-8859-15' => 'ISO-8859-15 (Latin-9, Western European revision with euro sign)',
+            'Windows-1251' => 'Windows-1251 (CP1251)',
+            'Windows-1252' => 'Windows-1252 (CP1252)'
+        ],
     ],
 ];

modules/backend/lang/fr/lang.php

@@ -358,5 +358,27 @@ return [
         'column_preview' => 'Prévisualisation des colonnes',
         'file_not_found_error' => 'Fichier non trouvé',
         'empty_error' => 'Il n‘y a aucune donnée à exporter',
+        'encoding_not_supported_error' => 'L’encodage de votre fichier source n’est pas reconnu. Veuillez sélectionner le format d’import personnalisé avec l’encodage adapté pour importer votre fichier.',
+        'encoding_format' => 'Encodage du fichier',
+        'encodings' => [
+            'utf-8' => 'UTF-8',
+            'us-ascii' => 'US-ASCII',
+            'iso-8859-1' => 'ISO-8859-1 (Latin-1, européen occidental)',
+            'iso-8859-2' => 'ISO-8859-2 (Latin-2, européen central)',
+            'iso-8859-3' => 'ISO-8859-3 (Latin-3, européen du Sud)',
+            'iso-8859-4' => 'ISO-8859-4 (Latin-4, européen du Nord)',
+            'iso-8859-5' => 'ISO-8859-5 (Latin, cyrillique)',
+            'iso-8859-6' => 'ISO-8859-6 (Latin, arabe)',
+            'iso-8859-7' => 'ISO-8859-7 (Latin, grec)',
+            'iso-8859-8' => 'ISO-8859-8 (Latin, hébreu)',
+            'iso-8859-0' => 'ISO-8859-9 (Latin-5, turc)',
+            'iso-8859-10' => 'ISO-8859-10 (Latin-6, nordique)',
+            'iso-8859-11' => 'ISO-8859-11 (Latin, thaï)',
+            'iso-8859-13' => 'ISO-8859-13 (Latin-7, balte)',
+            'iso-8859-14' => 'ISO-8859-14 (Latin-8, celtique)',
+            'iso-8859-15' => 'ISO-8859-15 (Latin-9, européen occidental révisé avec le signe euro)',
+            'Windows-1251' => 'Windows-1251 (CP1251)',
+            'Windows-1252' => 'Windows-1252 (CP1252)'
+        ],
     ],
 ];

modules/backend/models/ImportModel.php

@@ -87,10 +87,21 @@ abstract class ImportModel extends Model
      */
     protected function processImportData($filePath, $matches, $options)
     {
-        extract(array_merge([
+        /*
-            'firstRowTitles' => true
+         * Parse options
-        ], $options));
+         */
+        $defaultOptions = [
+            'firstRowTitles' => true,
+            'delimiter' => null,
+            'enclosure' => null,
+            'escape' => null
+        ];
 
+        $options = array_merge($defaultOptions, $options);
+
+        /*
+         * Read CSV
+         */
         $reader = CsvReader::createFromPath($filePath, 'r');
 
         // Filter out empty rows
@@ -98,7 +109,19 @@ abstract class ImportModel extends Model
             return count($row) > 1 || reset($row) !== null;
         });
 
-        if ($firstRowTitles) {
+        if ($options['delimiter'] !== null) {
+            $reader->setDelimiter($options['delimiter']);
+        }
+
+        if ($options['enclosure'] !== null) {
+            $reader->setEnclosure($options['enclosure']);
+        }
+
+        if ($options['escape'] !== null) {
+            $reader->setEscape($options['escape']);
+        }
+
+        if ($options['firstRowTitles']) {
             $reader->setOffset(1);
         }
 
@@ -165,6 +188,16 @@ abstract class ImportModel extends Model
         return $file->getLocalPath();
     }
 
+
+    /**
+     * Returns all available encodings values from the localization config
+     * @return array
+     */
+    public function getFormatEncodingOptions()
+    {
+        return \Lang::get('backend::lang.import_export.encodings');
+    }
+
     //
     // Result logging
     //