migrate previous lib/ (except tasks) to app/lib

app/lib/article_import.rb

@@ -0,0 +1,67 @@
+require 'digest/sha1'
+require 'tempfile'
+
+module ArticleImport
+
+  class ConversionFailedException < Exception; end
+
+  # return list of known file formats
+  #   each file_format module has
+  #   #name              return a human-readable file format name
+  #   #outlist_unlisted  if returns true, unlisted articles are outlisted
+  #   #detect            return a likelyhood (0-1) of being able to process
+  #   #parse             parse the data
+  #
+  def self.file_formats
+    @@file_formats ||= {
+      'bnn' => ArticleImport::Bnn,
+      'borkenstein' => ArticleImport::Borkenstein,
+      'foodsoft' => ArticleImport::Foodsoft,
+      'dnb_xml' => ArticleImport::DnbXml,
+      'bioromeo' => ArticleImport::Bioromeo,
+    }.freeze
+  end
+
+  # Parse file by type (one of {.file_formats})
+  #
+  # @param file [File, Tempfile]
+  # @option opts [String] type file format (required) (see {.file_formats})
+  # @return [File, Roo::Spreadsheet] file with encoding set if needed
+  def self.parse(file, type:, **opts, &blk)
+    # @todo handle wrong or undetected type
+    parser = file_formats[type]
+    if block_given?
+      parser.parse(file, **opts, &blk)
+    else
+      data = []
+      parser.parse(file, **opts) { |a| data << a }
+      data
+    end
+  end
+
+
+  # Helper method to generate an article number for suppliers that do not have one
+  def self.generate_number(article)
+    # something unique, but not too unique
+    s = "#{article[:name]}-#{article[:unit_quantity]}x#{article[:unit]}"
+    s = s.downcase.gsub(/[^a-z0-9.]/,'')
+    # prefix abbreviated sha1-hash with colon to indicate that it's a generated number
+    article[:number] = ':' + Digest::SHA1.hexdigest(s)[-7..-1]
+    article
+  end
+
+  # Helper method for opening a spreadsheet file
+  #
+  # @param file [File] file to open
+  # @param filename [String, NilClass] optional filename for guessing the file format
+  # @param encoding [String, NilClass] optional CSV encoding
+  # @param col_sep [String, NilClass] optional column separator
+  # @return [Roo::Spreadsheet]
+  def self.open_spreadsheet(file, filename: nil, encoding: nil, col_sep: nil)
+    opts = {csv_options: {}}
+    opts[:csv_options][:encoding] = encoding if encoding
+    opts[:csv_options][:col_sep] = col_sep if col_sep
+    opts[:extension] = File.extname(filename) if filename
+    Roo::Spreadsheet.open(file, **opts)
+  end
+end

app/lib/article_import/bioromeo.rb

@@ -0,0 +1,185 @@
+# Module for import of BioRomeo products from their Excel sheet, from Aug 2014 onwards
+
+require 'roo'
+require 'roo-xls'
+
+module ArticleImport::Bioromeo
+
+  NAME = "BioRomeo (XLSX, XLS, CSV)"
+  OUTLIST = true
+  OPTIONS = {
+    encoding: "UTF-8",
+    col_sep: ";"
+  }.freeze
+
+  RE_UNITS = /(kg|gr|gram|pond|st|stuks?|set|bos|bossen|bosjes?|bak|bakjes?|liter|ltr|[lL]\.|ml|bol|krop)(\s*\.)?/i
+  RES_PARSE_UNIT_LIST = [
+    /\b((per|a)\s*)?([0-9,.]+\s*x\s*[0-9,.]+\s*#{RE_UNITS})/i,                     # 1x5 kg
+    /\b((per|a)\s*)?([0-9,.]+\s*#{RE_UNITS}\s+x\s*[0-9,.]+)/i,                     # 1kg x 5
+    /\b((per|a)\s*)?(([0-9,.]+\s*,\s+)*[0-9,.]+\s+of\s+[0-9,.]+\s*#{RE_UNITS})/i,  # 1, 2 of 5 kg
+    /\b((per|a)\s*)?([0-9,.]+\s*#{RE_UNITS})/i,                                    # 1kg
+    /\b((per|a)\s*)?(#{RE_UNITS})/i                                                # kg
+  ]
+  # first parse with dash separator at the end, fallback to less specific
+  RES_PARSE_UNIT = RES_PARSE_UNIT_LIST.map {|r| /-\s*#{r}\s*$/} +
+                   RES_PARSE_UNIT_LIST.map {|r| /-\s+#{r}/} +
+                   RES_PARSE_UNIT_LIST.map {|r| /#{r}\s*$/} +
+                   RES_PARSE_UNIT_LIST.map {|r| /-#{r}/}
+
+  def self.parse(file, **opts)
+    opts = OPTIONS.merge(opts)
+    ss = ArticleImport.open_spreadsheet(file, **opts)
+
+    header_row = true
+    sheet = ss.sheet(0).parse(clean: true,
+      number:     /^artnr/i,
+      name:       /^product/i,
+      skal:       /^skal$/i,
+      demeter:    /^demeter$/i,
+      unit_price: /prijs\b.*\beenh/i,
+      pack_price: /prijs\b.*\bcolli/i,
+      comment:    /^opm(erking)?/i,
+    )
+
+    linenum = 0
+    category = nil
+
+    sheet.each do |row|
+      puts("[ROW] #{row.inspect}")
+      linenum += 1
+      row[:name].blank? and next
+      # (sub)categories are in first two content cells - assume if there's a price it's a product
+      if row[:number].blank? && row[:unit_price].blank?
+        category = row[:name]
+        next
+      end
+      # skip products without a number
+      row[:number].blank? and next
+      # extract name and unit
+      errors = []
+      notes = []
+      unit_price = row[:unit_price]
+      pack_price = row[:pack_price]
+      number = row[:number]
+      name = row[:name]
+      unit = nil
+      manufacturer = nil
+      prod_category = nil
+      RES_PARSE_UNIT.each do |re|
+        m=name.match(re) or next
+        unit = self.normalize_unit(m[3])
+        name = name.sub(re, '').sub(/\(\s*\)\s*$/,'').sub(/\s+/, ' ').sub(/\.\s*$/, '').strip
+        break
+      end
+      unit ||= '1 st' if name.match(/\bsla\b/i)
+      unit ||= '1 bos' if name.match(/\bradijs\b/i)
+      unit ||= '1 bosje' if category.match(/\bkruid/i)
+      if unit.nil?
+        unit = '?'
+        errors << "Cannot find unit in name '#{name}'"
+      end
+      # handle multiple units in one line
+      if unit.match(/\b(,\s+|of)\b/)
+        # TODO create multiple articles instead of taking first one
+      end
+      # sometimes category is also used to indicate manufacturer
+      m=category.match(/((eko\s*)?boerderij.*?)\s*$/i) and manufacturer = m[1]
+      # Ad-hoc fix for package of eggs: always take pack price
+      if name.match(/^eieren/i)
+        unit_price = pack_price
+        prod_category = 'Eieren'
+      end
+      prod_category = 'Kaas' if name.match(/^kaas/i)
+      # figure out unit_quantity
+      if unit.match(/x/)
+        unit_quantity, unit = unit.split(/\s*x\s*/i, 2)
+        unit,unit_quantity = unit_quantity,unit if unit_quantity.match(/[a-z]/i)
+      elsif (unit_price-pack_price).abs < 1e-3
+        unit_quantity = 1
+      elsif m=unit.match(/^(.*)\b\s*(st|bos|bossen|bosjes?)\.?\s*$/i)
+        unit_quantity, unit = m[1..2]
+        unit_quantity.blank? and unit_quantity = 1
+      else
+        unit_quantity = 1
+      end
+      # there may be a more informative unit in the line
+      if unit=='st' && !name.match(/kool/i)
+        RES_PARSE_UNIT.each do |re|
+          m=name.match(re) or next
+          unit = self.normalize_unit(m[3])
+          name = name.sub(re, '').strip
+        end
+      end
+      # note from various fields
+      notes.append("Skal #{row[:skal]}") if row[:skal].present?
+      notes.append(row[:demeter]) if row[:demeter].present? && row[:demeter].is_a?(String)
+      notes.append("Demeter #{row[:demeter]}") if row[:demeter].present? && row[:demeter].is_a?(Fixnum)
+      notes.append "(#{row[:comment]})" unless row[:comment].blank?
+      name.sub!(/(,\.?\s*)?\bDemeter\b/i, '') and notes.prepend("Demeter")
+      name.sub!(/(,\.?\s*)?\bBIO\b/i, '') and notes.prepend "BIO"
+      # unit check
+      errors << check_price(unit, unit_quantity, unit_price, pack_price)
+      # create new article
+      name.gsub!(/\s+/, ' ')
+      article = {:number => number,
+                 :name => name.strip,
+                 :note => notes.count > 0 && notes.map(&:strip).join("; "),
+                 :manufacturer => manufacturer,
+                 :origin => 'Noordoostpolder, NL',
+                 :unit => unit,
+                 :price => pack_price.to_f/unit_quantity.to_f,
+                 :unit_quantity => unit_quantity,
+                 :tax => 6,
+                 :deposit => 0,
+                 :category => prod_category || category
+                 }
+      errors.compact!
+      if errors.count > 0
+        yield article, errors.join("\n")
+      else
+        # outlisting not used by supplier
+        yield article, nil
+      end
+    end
+  end
+
+  protected
+
+  def self.check_price(unit, unit_quantity, unit_price, pack_price)
+    if (unit_price-pack_price).abs < 1e-3
+      return if unit_quantity == 1
+      return "price per unit #{unit_price} is pack price, but unit quantity #{unit_quantity} is not one"
+    end
+
+    if m = unit.match(/^(.*)(#{RE_UNITS})\s*$/)
+      amount, what = m[1..2]
+    else
+      return "could not parse unit: #{unit}"
+    end
+
+    # perhaps unit price is kg-price
+    kgprice = if what =~ /^kg/i
+                pack_price.to_f / amount.to_f
+              elsif what =~ /^gr/
+                pack_price.to_f / amount.to_f * 1000
+              end
+    if kgprice.present? && (kgprice - unit_price.to_f).abs < 1e-2
+      return
+    end
+
+    unit_price_computed = pack_price.to_f/unit_quantity.to_i
+    if (unit_price_computed - unit_price.to_f).abs > 1e-2
+      "price per unit given #{unit_price.round(3)} does not match computed " +
+        "#{pack_price.round(3)}/#{unit_quantity}=#{unit_price_computed.round(3)}" +
+        (kgprice ? " (nor is it a kg-price #{kgprice.round(3)})" : '')
+    end
+  end
+
+  def self.normalize_unit(unit)
+    unit = unit.sub(/1\s*x\s*/, '')
+    unit = unit.sub(/,([0-9])/, '.\1').gsub(/^per\s*/,'').sub(/^1\s*([^0-9.])/,'\1').sub(/^a\b\s*/,'')
+    unit = unit.sub(/(bossen|bosjes?)/, 'bos').sub(/(liter|l\.|L\.)/,'ltr').sub(/stuks?/, 'st').sub('gram','gr')
+    unit = unit.sub(/\s*\.\s*$/,'').sub(/\s+/, ' ').strip
+  end
+
+end

app/lib/article_import/bnn.rb

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+require 'csv'
+require 'yaml'
+
+# Module for translation and parsing of BNN-files (www.n-bnn.de)
+#
+module ArticleImport::Bnn
+
+ private
+  @@codes = Hash.new
+  @@midgard = Hash.new
+
+  # Loads the codes_file config/bnn_codes.yml into the class variable @@codes
+  def self.load_codes
+    dir = Rails.root.join("app", "lib", "article_import")
+    begin
+      @@codes = YAML::load(File.open(dir.join("bnn_codes.yml"))).symbolize_keys
+      @@midgard = YAML::load(File.open(dir.join("midgard_codes.yml"))).symbolize_keys
+    rescue => e
+      raise "Failed to load bnn_codes: #{dir}/{bnn,midgard}_codes.yml: #{e.message}"
+    end
+  end
+
+ public
+  $missing_bnn_codes = Array.new
+
+  # translates codes from BNN to foodsoft-code
+  def self.translate(key, value)
+    if @@codes[key][value]
+      return @@codes[key][value]
+    elsif @@midgard[key]
+      return @@midgard[key][value]
+    elsif value != nil
+      $missing_bnn_codes << value
+      return nil
+    end
+  end
+  
+  NAME = "BNN (CSV)"
+  OUTLIST = false
+  OPTIONS = {
+    encoding: "IBM850",
+    col_sep: ";"
+  }.freeze
+
+  # parses a bnn-file
+  def self.parse(file, **opts)
+    encoding = opts[:encoding] || OPTIONS[:encoding]
+    col_sep = opts[:col_sep] || OPTIONS[:col_sep]
+    CSV.foreach(file, {col_sep: col_sep, encoding: encoding, headers: true}) do |row|
+      # check if the line is empty
+      unless row[0] == "" || row[0].nil?
+        article = {
+            :name => row[6],
+            :number => row[0],
+            :note => row[7],
+            :manufacturer => self.translate(:manufacturer, row[10]),
+            :origin => row[12],
+            :category => self.translate(:category, row[16]),
+            :unit => row[23],
+            :price => row[37],
+            :tax => self.translate(:tax, row[33]),
+            :unit_quantity => row[22]
+        }
+        # TODO: Complete deposit list....
+        article.merge!(:deposit => self.translate(:deposit, row[26])) if self.translate(:deposit, row[26])
+
+        # get scale prices if exists
+        article.merge!(:scale_quantity => row[40], :scale_price => row[41]) unless row[40].nil? or row[41].nil?
+
+        if row[62] != nil
+          # consider special prices
+          article[:note] = "Sonderpreis: #{article[:price]} von #{row[62]} bis #{row[63]}"
+          yield article, :special
+
+          # Check now for article status, we only consider outlisted articles right now
+          # N=neu, A=Änderung, X=ausgelistet, R=Restbestand,
+          # V=vorübergehend ausgelistet, W=wiedergelistet
+        elsif row[1] == "X" || row[1] == "V"
+          yield article, :outlisted
+        else
+          yield article, nil
+        end
+      end
+    end
+  end
+end
+
+# Automatically load codes:
+ArticleImport::Bnn.load_codes

app/lib/article_import/borkenstein.rb

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+# Module for Borkenstein csv import
+
+require 'csv'
+
+module ArticleImport::Borkenstein
+
+  REGEX = {
+    :main => /^(.+)\s+\[([^\[\]]+)\]\s+(\d+\.\d+)\((\d+\.\d+)\)$/,
+    :manufacturer => /^(.+)\s{4}\[\]\s{4}\(\)$/,
+    :origin => /(.+)\s+(\w+)\/\w+[\/[\w\-]+]?/
+  }.freeze
+  
+  NAME = "Borkenstein (CSV)"
+  OUTLIST = false
+  OPTIONS = {
+    col_sep: ",",
+    encoding: "UTF-8" # @todo check this
+  }.freeze
+
+  def self.parse(file, **opts)
+    global_manufacturer = nil
+
+    file.set_encoding(opts[:encoding] || OPTIONS[:encoding])
+    col_sep = opts[:col_sep] || OPTIONS[:col_sep]
+    CSV.new(file, {col_sep: col_sep, :headers => false}).each do |row|
+
+      # Set manufacturer
+      if row[1] == "-"
+        match = row[2].match(REGEX[:manufacturer])
+        global_manufacturer = match.captures.first unless match.nil?
+      end
+
+      # check if the line is empty
+      unless row[1].blank? || row[1] == "-"
+
+        # Split string and remove beginning "
+        matched = row[2].gsub(/^\"/, "").gsub(/\"$/, "").match(REGEX[:main])
+
+        if matched.nil?
+          puts "No regular article data for #{row[1]}: #{row[2]}"
+
+        else
+
+          name, units, price_high, price_low = matched.captures
+
+          # Try to get origin
+          matched_name = name.match(REGEX[:origin])
+          if matched_name
+            name, origin = matched_name.captures
+          else
+            name, origin = name.gsub(/\s{2,}/, ""), nil
+          end
+
+          # Manufacturer
+          if name.match(/^[A-Za-z]{2,3}\s{1}/)
+            name.gsub!(/^[A-Za-z]{2,3}\s{1}/, "")
+            manufacturer = global_manufacturer
+          end
+
+
+          # Get unit quantities
+          units = units.split("x")
+          if units.size == 2
+            unit_quantity = units.first
+            unit = units.last
+          else
+            unit_quantity = 1
+            unit = units.first
+          end
+
+          article = {
+            :number => row[1],
+            :name => name,
+            :origin => origin,
+            :manufacturer => manufacturer,
+            :unit_quantity => unit_quantity,
+            :unit => unit,
+            :price => price_low, # Inklusive Rabattstufe von 10%
+            :tax => 0.0 # Tax is included
+          }
+
+          # test, if neccecary attributes exists
+          if article[:unit].nil? || article[:price].nil? || article[:unit_quantity].nil?
+            raise "Fehler: Einheit, Preis und MwSt. müssen gegeben sein: #{article.inspect}"
+          end
+
+          yield article, nil
+        end
+      end
+    end
+  end
+
+end

app/lib/article_import/dnb_codes.yml

@@ -0,0 +1,130 @@
+
+# from http://www.nieuweband.nl/producten/groepen/
+indeling:
+  1: 'Verswaren'
+  50: 'Kaas'
+  62: 'Schapenkaas'
+
+  2: 'Basisproducten'
+  850: 'Noten'
+  855: 'Noten grootverbruik'
+  700: 'Peulvruchten'
+  705: 'Peulvruchten grootverbruik'
+  340: 'Rijst'
+  341: 'Rijst grootverbruik'
+  450: 'Vlokken'
+  455: 'Vlokken grootverbruik'
+  800: 'Zaden en pitten'
+  805: 'Zaden en pitten grootverbruik'
+  603: 'Melen grootverbruik'
+
+  3: 'Ontbijt en lunch'
+  943: 'Marmelade'
+  1272: 'Muesli en poppies'
+  1000: 'Notenpasta'
+  1276: 'Ontbijtmelen'
+  1295: 'Rijstwafels'
+  1290: 'Roggebrood'
+  1270: 'Sandwichspread'
+  940: 'Vruchtenbeleg'
+  942: 'Vruchtenjam'
+  944: 'Vruchtenstroop'
+  1300: 'Knäckebröd, toast en beschuit'
+
+  4: 'Warme maaltijd'
+  1820: 'Mosterd'
+  1610: 'Olijfolie'
+  1600: 'Olijven'
+  1451: 'Peulvruchtenconserven'
+  1957: 'Pindasaus'
+  1960: 'Sambal, ketjap en pittige smaakmakers'
+  2170: 'Seitan'
+  2260: 'Siropen'
+  2248: 'Smaakmakers'
+  1500: 'Soepen en bouillon'
+  1515: 'Soepstengels'
+  2000: 'Sojasauzen'
+  2250: 'Suiker'
+  1452: 'Tafelzuren'
+  1590: 'Tamme-kastanje-producten'
+  1975: 'Thaise keuken'
+  1900: 'Tomatenproducten'
+  1670: 'Vetten'
+  1930: 'Visconserven'
+  2175: 'Vleesvervangers'
+  1360: 'Vruchtencompote'
+  1400: 'Vruchtenconserven'
+  1350: 'Vruchtenmoes en -puree'
+  2249: 'Zout en kruidenzout'
+
+  5: 'Sappen en dranken'
+  2605: 'Rode wijn Oostenrijk'
+  2604: 'Rode wijn Portugal'
+  2602: 'Rode wijn Spanje'
+  2608: 'Rode wijn Zuid-Afrika'
+  2612: 'Rosé Spanje'
+  2617: 'Rosé Zuid-Afrika'
+  2420: 'Smoothies'
+  2455: 'Sojamelkproducten'
+  2505: 'Speciaalbieren'
+  2400: 'Vruchtensappen'
+  2490: 'Waterijs'
+  2637: 'Witte wijn Argentinië'
+  2630: 'Witte wijn Frankrijk'
+  2634: 'Witte wijn Griekenland'
+  2631: 'Witte wijn Italië'
+  2635: 'Witte wijn Oostenrijk'
+  2632: 'Witte wijn Spanje'
+  2638: 'Witte wijn Zuid-Afrika'
+
+  6: 'Warme dranken en theekruiden'
+  3102: 'Kruidenthee builtjes'
+  3100: 'Kruidenthee los'
+  3020: 'Kruidenthee met geneeskrachtige werking'
+  3009: 'Rooibosthee'
+  3010: 'Thee grootverpakking'
+  3052: 'Theekruiden'
+  3008: 'Witte thee'
+  3011: 'Yogi spice tea'
+  3012: 'Yogi tao tea'
+  3000: 'Zwarte thee'
+
+  7: 'Versnaperingen'
+  3552: 'Lollies'
+  3470: 'Nougat en fudge'
+  3570: 'Raw Food'
+  3360: 'Rozijntjes in kinderverpakking'
+  3410: 'Snijkoek'
+  3555: 'Snoep met suiker'
+  3550: 'Snoep zonder suiker'
+  3405: 'Stroopwafels'
+  3350: 'Tortillachips en salsa'
+  3358: 'Zoete chips'
+  3540: 'Zoethoutstokjes'
+  3365: 'Zoutjes, hartige bites en popcorn'
+  3530: 'Laurierdrop'
+
+  8: 'Persoonlijke verzorging en cosmetica'
+  5036: 'Lavera'
+  5037: 'Namaste'
+  5040: 'Natracare'
+  5042: 'Odylique'
+  5049: 'Sonett'
+  5055: 'Urtekram'
+  5065: 'Weleda'
+
+  9: 'Natuurtherapeutisch'
+  5455: 'Kruidentincturen'
+  5420: 'Propolis-producten'
+  5245: 'Zelfzorgmiddelen'
+  5280: 'Huid- en massage-olie'
+
+  10: 'Non Food'
+  5517: 'Luiers en babydoekjes'
+  5510: 'Maandverband en tampons'
+  5520: 'Toiletpapier e.d.'
+  5890: 'Voor kinderen (en volwassenen)'
+  5650: 'Was- en schoonmaakmiddelen'
+  5515: 'Watten'
+  5610: 'Luchtverfrissers'
+

app/lib/article_import/dnb_xml.rb

@@ -0,0 +1,73 @@
+# Article import for De Nieuw Band XML file
+#
+# Always contains full assortment, including recently outlisted articles.
+# To make sure we don't keep old articles when a number of updates was missed,
+# +OUTLIST+ is set to +true+ to remove articles not present in the file.
+#
+require 'nokogiri'
+
+module ArticleImport::DnbXml
+
+  NAME = "De Nieuwe Band (XML)"
+  OUTLIST = true
+  OPTIONS = {}.freeze
+
+  # parses a string or file
+  def self.parse(file, opts={})
+    doc = Nokogiri.XML(file, nil, nil,
+      Nokogiri::XML::ParseOptions::RECOVER +
+      Nokogiri::XML::ParseOptions::NONET +
+      Nokogiri::XML::ParseOptions::COMPACT # do not modify doc!
+    )
+    doc.search('product').each do |row|
+      # create a new article
+      unit = row.search('eenheid').text
+      unit = case(unit)
+        when blank? then 'st'
+        when 'stuk' then 'st'
+        when 'g'    then 'gr' # need at least 2 chars
+        when 'l'    then 'ltr'
+        else             unit
+      end
+      inhoud = row.search('inhoud').text
+      inhoud.blank? or (inhoud.to_f-1).abs > 1e-3 and unit = inhoud.gsub(/\.0+\s*$/,'') + unit
+      deposit = row.search('statiegeld').text
+      deposit.blank? and deposit = 0
+      category = [
+        @@codes[:indeling][row.search('indeling').text.to_i],
+        @@codes[:indeling][row.search('subindeling').text.to_i]
+      ].compact.join(' - ')
+
+      article = {:number => row.search('bestelnummer').text,
+                 #:ean => row.search('eancode').text,
+                 :name => row.search('omschrijving').text,
+                 :note => row.search('kwaliteit').text,
+                 :manufacturer => row.search('merk').text,
+                 :origin => row.search('herkomst').text,
+                 :unit => unit,
+                 :price => row.search('prijs inkoopprijs').text,
+                 :unit_quantity => row.search('sve').text,
+                 :tax => row.search('btw').text,
+                 :deposit => deposit,
+                 :category => category}
+
+      yield article, (row.search('status') == 'Actief' ? :outlisted : nil)
+    end
+  end
+
+  private
+
+  @@codes = Hash.new
+
+  def self.load_codes
+    dir = Rails.root.join("lib", "article_import")
+    begin
+      @@codes = YAML::load(File.open(dir.join("dnb_codes.yml"))).symbolize_keys
+    rescue => e
+      raise "Failed to load dnb_codes: #{dir}/dnb_codes.yml: #{e.message}"
+    end
+  end
+
+end
+
+ArticleImport::DnbXml.load_codes

app/lib/article_import/foodsoft.rb

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+# Module for Foodsoft-file import
+# The Foodsoft-file is a CSV-file, with semicolon-separated columns, or ODS/XLS/XLSX
+
+require 'roo'
+require 'roo-xls'
+
+module ArticleImport::Foodsoft
+
+  NAME = "Foodsoft (CSV, ODS, XLS, XLSX)"
+  OUTLIST = false
+  OPTIONS = {
+    encoding: "UTF-8",
+    col_sep: ";"
+  }.freeze
+
+  # Parses Foodsoft file
+  # the yielded article is a simple hash
+  def self.parse(file, **opts)
+    opts = OPTIONS.merge(opts)
+    ss = ArticleImport.open_spreadsheet(file, **opts)
+
+    header_row = true
+    ss.sheet(0).each do |row|
+      # skip first header row
+      if header_row
+        header_row = false
+        next
+      end
+      # skip empty lines
+      next if row[2].blank?
+
+      article = {:number => row[1],
+                 :name => row[2],
+                 :note => row[3],
+                 :manufacturer => row[4],
+                 :origin => row[5],
+                 :unit => row[6],
+                 :price => row[7],
+                 :tax => row[8],
+                 :unit_quantity => row[10],
+                 :scale_quantity => row[11],
+                 :scale_price => row[12],
+                 :category => row[13]}
+      article.merge!(:deposit => row[9]) unless row[9].nil?
+      article[:number].blank? and ArticleImport.generate_number(article)
+      if row[6].nil? || row[7].nil? or row[8].nil?
+        yield article, "Error: unit, price and tax must be entered"
+      else
+        yield article, (row[0]=='x' ? :outlisted : nil)
+      end
+    end
+  end
+
+end

app/lib/article_import/midgard_codes.yml

@@ -0,0 +1,294 @@
+manufacturer:
+  "61": Maintal
+  AB: Agrobioservice
+  AD: Anita Dehnert
+  AH: Phyto Treasures e.K.
+  AO: Arganöl
+  AR: ARIES
+  AS: Abraham Schinken
+  Ad: Molkerei Andechs
+  An: Frans Andringa
+  Ap: Apfeltraum
+  Ar: Provamel über Arche
+  Ay: Aytem
+  BA: BauckHof Amelinghausn
+  BB: Bakenhus Biofleisch GmbH
+  BC: Bio-Bäckerei Bucco
+  BD: Biosa
+  BF: Bruno Fischer
+  BG: Bauers Garten
+  BH: Bauck Hof
+  BHA: Bauck Hof Amelinghausen
+  BI: Biofarben
+  BK: Burger Knäcke
+  BKO: BioKräuterei Oberhavel
+  BL: Beumer & Lutum
+  BM: Bohlsener Mühle
+  BN: Brochenin
+  BOD: Bode Naturkost
+  BR: Luchs Bier
+  BT: Beltane Naturkost GmbH
+  BU: Baumschule am Butzelberg
+  BV: BIO VITA
+  BZ: Biozeit
+  Ba: Bauck demeter Produkte
+  Bb: Beutelsbacher
+  Bd: Biosa Danmark Aps
+  Be: Behncken
+  Bf: Backforum
+  Bg: Butzelberg
+  Bh: Barnhouse
+  Bj: Milchschafhof Brünjes
+  Bk: Blank
+  Bm: Biomax
+  Bn: Bentele
+  Bo: Bobalis
+  Bt: Bretti's
+  Bu: Biogärtnerei Bauer
+  By: Byodo
+  CA: Care
+  CF: CLUB Feinkost
+  CI: CIDRERIE
+  CV: Cosmoveda
+  Ca: Campo
+  Cl: Obsthof Clostermann
+  Co: Obsthof Cordes (Heinrich)
+  Cp: Campobello
+  Cs: Cosmoveda
+  Ct: cbet GmbH
+  DA: Danival
+  DE: DEMETER-Erzeugergemeinschaft
+  DH: Dieter Hein Wurstwaren
+  DM: Dr. Martins
+  DN: Hof Dannwisch
+  DO: Donath-Mühle
+  DR: De Rit
+  DV: Davert
+  DW: Vovic / Evian
+  De: Dennree
+  Dk: Dinkula
+  EB: Erich Boden
+  EH: Engemann Handel
+  EI: Natürlich Eistert
+  ELM: BIONADE
+  EN: Provence Regime
+  EO: Eosta
+  ER: Euresis
+  Eb: Eisblümerl
+  Eh: Erhardt Meerrettichprodukte
+  Ei: Eiland
+  El: Kelterei Elm
+  En: Eichhorn
+  Er: Erdmannhauser Brezelfabrik
+  Es: Erntesegen
+  FB: Flensburger Brauerei
+  FE: Frucht-Express
+  FF: Schiffers
+  FI: Fromi GmbH
+  FL: Florian Kerzen
+  FR: I Frutti del Sole
+  FU: Future 3000
+  Fh: Florahof
+  Fq: Fläming-Quelle
+  Fr: Frunet
+  Ft: Fontaine
+  GA: Bio-Gärtnerei Altglobsow
+  GG: Naturhof Günter Gaßmann
+  GH: Gutshöfe
+  GN: Nesse Gewürze
+  GO: Der Georgshof
+  GS: Gut Schmerwitz
+  GT: Gut Temmen
+  Gb: Grabower
+  Gl: Glaciar
+  Go: Golden Temple
+  Gr: Grützdorfer
+  Gw: Gwidon Zastawa
+  Gä: Gärtnerei am Bauerngut
+  GÖ: Stadtgut Görlitz
+  HA: Haaner Felsenquelle
+  HB: Hof Bockum
+  HF: Hühnerhof Falkenthal
+  HK: Heinz Ketchup
+  HM: Hof Marienhöhe
+  HO: Hoffmann
+  HS: Obstbau H. Schalkau
+  Ha: Hake
+  Hc: Hoch Oblatenfabrik
+  He: Hennicke
+  Hk: Natur Obsthof Hauke
+  Hl: Heidehof
+  Ho: Holle
+  Hu: Humanopolis
+  Hü: Hütterman
+  IC: Japan Grüntee
+  IN: Isola della Natura
+  IOC: IOC
+  IS: Isana
+  Ib: Iberia
+  Il: Il Nuraghe
+  Is: ISANA
+  JH: Beerenobst
+  JS: Juers Fruchtchips
+  Je: Jelitta Käse
+  KD: Kristdyn
+  KG: Kräuter Gut
+  KK: 74271
+  KN: Öko-Gartenbau
+  KP: Kräutergarten Pommerland
+  Ka: Kanne
+  Kg: Karg Brotgenuß
+  Kä: Kärrners
+  Kö: Obsthof König
+  LB: Lammsbräu
+  LE: LEEB Schaf- und Ziegenmolkerei
+  LI: Legend Organics
+  LM: LeMar
+  LS: La Selva
+  La: Lahmann
+  Lb: Lebensbaum
+  Le: Leuchtenberg Sauerkrautfabrik
+  Lh: Lindenhof
+  Li: Lima Belgien
+  Lk: Landkrone
+  Ln: Land in Sicht
+  Ls: Lubs GmbH
+  Lu: Luvos Heilerde
+  Lw: Gärtnerei Löwenzahn
+  MA: Mack
+  MB: Mabutake
+  ME: Martin Evers
+  MH: Märkische Heide
+  MI: Martin Ibele
+  MII: Katal. Olivenöl
+  ML: Märkisches Landbrot
+  MM: Bioland Imkerei
+  MT: Maintal
+  MV: MegaVega Limited
+  MY: Mayka Brezel
+  Ma: Marschland
+  Mg: MIDGARD
+  Mh: Melchhof
+  Mn: Mosna
+  Mo: Mosaikwerkstätten
+  My: MAYKA, Brezelfabrik
+  Mü: Hofmolkerei GmbH Münchehofe
+  MÖ: Märkischer Ökovertrieb
+  NE: Natürlich Eistert
+  NM: 
+  NO: Nürnberger Bio Originale
+  NQ: Pineo Wasser
+  Na: NATURATA
+  Nt: Natumi
+  OTC: OTC
+  Od: ODIN Holland
+  PB: Peter Bentele
+  PG: Pilzgarten
+  PH: Biopilzhof
+  PM: Pinkus Müller
+  PN: Pro Natura
+  Pi: Piding
+  Pt: Port International
+  QB: Panettoncino
+  RB: Rother Bräu
+  RP: Rheinsberger Preussenquelle
+  RS: rosmarin BIOBACK
+  RZ: Ranch Zempow
+  Ra: Raab
+  Rb: Rabenhorst
+  Re: Rebgarten
+  Rg: Rosengarten
+  Rh: Rotenhäusler
+  Ro: Geflügelhof Robert
+  RoL: Robert´s LOSE
+  Rt: Rottstock
+  Rö: Römerquelle
+  SB: Sabines Bauernhof
+  SBP: Stiftelsen Bananen
+  SC: Sommer & Co.
+  SF: Sprossen
+  SH: Spreewälder Hirse
+  SI: SINFO
+  SK: Spargelhof Kreienbaum
+  SL: St. Leonhardsquelle
+  SM: Seenlan Müritz
+  SO: Sonett
+  SR: Sprossenmanufaktur GbR
+  STN: Sonnentor
+  SV: SANTAVERDE ALOE VERA
+  Sa: Salamita
+  Sb: Hans Hermann Soetbeer
+  Sc: Schulz-Deetz
+  Sch: Hof Schütte
+  Sd: Savid
+  Se: Sekem, Ägypten
+  Sf: Sauerkonservenfabrik Schweizer
+  Sh: Kombucha
+  Si: Land in Sicht
+  Sk: Schock       Ludwigsburg
+  Sm: Schramm
+  So: Sophienhof
+  Sp: Spielberger
+  Sr: Sanmar
+  St: Steck Senf
+  StB: Stralsunder Brauerei
+  Su: Sun,Backwaren aus Norwegen
+  Sv: Sunval demeter-Produkte
+  Sw: Szilleweit
+  Sy: Synanon
+  Sz: Schrozberg
+  Sü: Südasien
+  TB: Team Blue
+  TF: Terra Frischdienst
+  TN: Tofu Nagel
+  TR: Teltower Rübchen
+  Ta: Tarpa
+  Te: Teutoburger Ölmühle
+  Ti: Tiedemann
+  Tm: Tillmann
+  Tr: tri d´Aix
+  Tt: Tautropfen
+  Tö: Töpfer Rohrzucker
+  UK: Udo Kolm Bananen
+  UL: Gärtnerei Ulenburg
+  UV: Uni-Vert
+  Ul: Ulenburg Bioland Gemüse
+  VA: Kleingenossenschaft VENUSTA
+  VD: V & D
+  VE: Vega e.K.
+  VG: Biolog. Vollwertgetränke
+  VT: Vogt
+  VV: Vallé Käse
+  Vi: Viana Tofu
+  VlV: Vivo Lo Vin
+  Vo: Voelkel
+  WB: Weber
+  WD: Werder Feinkost GmbH
+  WH: Weide-Hardebek
+  WK: BioCompany Kaffee
+  WL: Wendland Storchenmilch
+  WP: Plosewasser
+  WR: Speickwerke
+  WS: Weingut Sander
+  Wa: Watzkendorf
+  We: Wendts
+  Wh: Molkerei Weißenhorn
+  Wz: Werz       Heidenheim
+  ZA: Bio-Center Zann
+  ZF: Obsthof zum Felde
+  ZG: Zwergenwiese
+  ZI: Biolandhof Zielke
+  ZK: Ziegenkäserei Karolinenhof
+  ZP: Bioland Ranch Zempow
+  ZW: Zellertaler Wein
+  bF: bio Frische
+  bi: biosanica
+  dB: ÖMA-d`Beers, Kisslegg im Algäu
+  eu: felicia
+  fa: familia Müsli
+  ha: Hawlik
+  vL: v.d.Linden
+  öG: Öko-Gartenbau
+  öh: ökohum Blumenerde
+  ÖL: Öko-Line
+  ÖS: Ölmühle Solling

app/lib/ftp_sync.rb

@@ -0,0 +1,36 @@
+require 'net/ftp'
+require 'fileutils'
+
+module FtpSync
+
+  # compares remote with local filelist
+  # if local file not exists or older than remote file, download remote file
+  # return array with new files
+  def self.sync(supplier)
+    new_files = Array.new
+
+    # change local dir to save files correctly
+    FileUtils.mkdir_p(supplier.ftp_path) unless File.exists?(supplier.ftp_path)
+    Dir.chdir(supplier.ftp_path)
+
+    # connect to ftp-server
+    ftp = Net::FTP.new(supplier.ftp_host, supplier.ftp_user, supplier.ftp_password)
+
+    # loop over the remote filelist
+    ftp.nlst.each do |filename|
+      if filename.match(Regexp.new(supplier.ftp_regexp))
+        # local file not exist or remote file newer ?
+        if (File.exist?(filename) and File.new(filename).mtime < ftp.mtime(filename)) or !File.exist?(filename)
+          # download
+          ftp.getbinaryfile(filename)
+          # save filename for return
+          new_files << filename
+        end
+      end
+    end
+    # close ftp-session
+    ftp.close
+    return new_files
+  end
+
+end

app/models/supplier.rb

@@ -7,7 +7,7 @@ class Supplier < ActiveRecord::Base
   serialize :lists
 
   FTP_TYPES = ['bnn', 'foodsoft'].freeze
-  EMAIL_RE = /^([^@\s]+)@((?:[-a-z0-9]+\.)+[a-z]{2,})$/i.freeze
+  EMAIL_RE = /\A([^@\s]+)@((?:[-a-z0-9]+\.)+[a-z]{2,})\z/i.freeze
 
   validates :name, :address, :phone, presence: true
   validates :ftp_host, :ftp_user, :ftp_password, :ftp_sync, presence: true, if: :ftp_sync?