Fixing "Wrong line format" issue

.dockerignore

@@ -5,6 +5,7 @@ docker-compose.yml
 config.yml
 config.sample.yml
 
+.git
 .cache
 .export
 last_imported

.travis.yml

@@ -2,7 +2,8 @@ language: ruby
 cache: bundler
 rvm:
 - 2.4.2
-- 2.5.0
+- 2.5.3
+- 2.6.1
 env:
   - ENV=test
 before_script:

Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:2.5.0
+FROM ruby:2.6
 
 # Set the locale
 RUN apt-get clean && apt-get update && apt-get install -y locales

Gemfile.lock

@@ -1,86 +1,90 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.1.5)
+    activesupport (5.2.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (~> 0.7)
+      i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    addressable (2.5.2)
+    addressable (2.6.0)
       public_suffix (>= 2.0.2, < 4.0)
     ast (2.4.0)
-    bankscrap (2.0.6)
+    bankscrap (2.1.1)
       activesupport
+      json
       mechanize
       money
       thor
       unicode-display_width
     bankscrap-bbva (2.0.3)
       bankscrap (>= 2.0.0)
     base32 (0.3.2)
-    cmxl (0.2.1)
+    cmxl (0.2.2)
       rchardet19
     colorize (0.8.1)
-    concurrent-ruby (1.0.5)
+    concurrent-ruby (1.1.4)
+    connection_pool (2.2.2)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     deep_merge (1.2.1)
     diff-lcs (1.3)
-    domain_name (0.5.20170404)
+    domain_name (0.5.20180417)
       unf (>= 0.0.5, < 1.0.0)
-    ethon (0.11.0)
+    ethon (0.12.0)
       ffi (>= 1.3.0)
-    ffi (1.9.25)
-    hashdiff (0.3.7)
+    ffi (1.10.0)
+    hashdiff (0.3.8)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
-    httparty (0.16.0)
+    httparty (0.16.3)
+      mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
-    i18n (0.9.5)
+    i18n (1.5.3)
       concurrent-ruby (~> 1.0)
     json (2.1.0)
-    mechanize (2.7.5)
+    mechanize (2.7.6)
       domain_name (~> 0.5, >= 0.5.1)
       http-cookie (~> 1.0)
       mime-types (>= 1.17.2)
       net-http-digest_auth (~> 1.1, >= 1.1.1)
-      net-http-persistent (~> 2.5, >= 2.5.2)
+      net-http-persistent (>= 2.5.2)
       nokogiri (~> 1.6)
       ntlm-http (~> 0.1, >= 0.1.1)
       webrobots (>= 0.0.9, < 0.2)
-    mime-types (3.1)
+    mime-types (3.2.2)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2016.0521)
-    mini_portile2 (2.3.0)
+    mime-types-data (3.2018.0812)
+    mini_portile2 (2.4.0)
     minitest (5.11.3)
-    money (6.10.1)
-      i18n (>= 0.6.4, < 1.0)
+    money (6.13.2)
+      i18n (>= 0.6.4, <= 2)
     multi_xml (0.6.0)
     net-http-digest_auth (1.4.1)
-    net-http-persistent (2.9.4)
-    nokogiri (1.8.2)
-      mini_portile2 (~> 2.3.0)
+    net-http-persistent (3.0.0)
+      connection_pool (~> 2.2)
+    nokogiri (1.10.1)
+      mini_portile2 (~> 2.4.0)
     ntlm-http (0.1.1)
-    parallel (1.12.1)
-    parser (2.5.0.2)
+    parallel (1.13.0)
+    parser (2.6.0.0)
       ast (~> 2.4.0)
-    powerpack (0.1.1)
-    public_suffix (3.0.2)
+    powerpack (0.1.2)
+    public_suffix (3.0.3)
     rainbow (3.0.0)
     rchardet19 (1.3.7)
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.1)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-mocks (3.7.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-support (3.7.1)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
     rubocop (0.52.1)
       parallel (~> 1.10)
       parser (>= 2.4.0.2, < 3.0)
@@ -90,26 +94,26 @@ GEM
       unicode-display_width (~> 1.0, >= 1.0.1)
     rubocop-rspec (1.23.0)
       rubocop (>= 0.52.1)
-    ruby-progressbar (1.9.0)
+    ruby-progressbar (1.10.0)
     ruby_fints (0.0.3)
       cmxl (~> 0.2)
       httparty (~> 0.10)
     safe_yaml (1.0.4)
-    thor (0.20.0)
+    thor (0.20.3)
     thread_safe (0.3.6)
     twentysix (0.1.1)
       deep_merge
       httparty
-    typhoeus (1.3.0)
+    typhoeus (1.3.1)
       ethon (>= 0.9.0)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.5)
-    unicode-display_width (1.3.0)
+    unicode-display_width (1.4.1)
     vcr (4.0.0)
-    webmock (3.3.0)
+    webmock (3.5.1)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff
@@ -138,4 +142,4 @@ DEPENDENCIES
   ynab (= 1.5.0)
 
 BUNDLED WITH
-   1.16.2
+   1.17.3

README.md

@@ -11,6 +11,12 @@ This is a ruby script that **pulls your transactions from your banks** and impor
 
 ---
 
+## Disclaimer
+
+**Using this tool is on your own risk.** I can **not** garantee you that this will work for you. Also I can **not** garantee that it won't up your YNAB data. I highly recommend to start with a new Budget or at least test it with a new one before you use your existing Budget.
+
+---
+
 ## Supported banks
 
 * Most German and Austrian banks _(all banks that implement the FinTS standard)_
@@ -32,17 +38,21 @@ The script also includes some additional logic like detecting internal transacti
 # Known Problems
 
 * Currently no known problems
+* Please read the notes in each Dumper _[(see Wiki)](https://github.com/schurig/ynab-bank-importer/wiki#supported-dumpers)_ to understand the limitations
 
 ____________________
 
 # Support / Contribution
 
 Support and contriubution of any kind is always welcome!!!
 
-I'm not that into hardware. It would be super awesome if someone could help making this work on Raspbian. I already tried but building the docker container fails _(Dockerfile.rpi)_. The PR related to that you can find here: [18](https://github.com/schurig/ynab-bank-importer/pull/18)
-
 # Thanks
 
+* [@dequis](https://github.com/dequis) for preventing the script from failing in the future [PR #70](https://github.com/schurig/ynab-bank-importer/pull/70) and keeping an eye on the development experience [PR #69](https://github.com/schurig/ynab-bank-importer/pull/69)
+* [@moay](https://github.com/moay), [@yuvke](https://github.com/yuvke), [@BluetriX](https://github.com/BluetriX) for the help on debugging an error with not existing dates [Issue #52](https://github.com/schurig/ynab-bank-importer/issues/52)
+* [@mathijshoogland](https://github.com/mathijshoogland) for updating the dependencies [PR #49](https://github.com/schurig/ynab-bank-importer/pull/49)
+* [@manuelgrabowski](https://github.com/manuelgrabowski), [@martinlabuschin](https://github.com/martinlabuschin) and [@peterjeschke](https://github.com/peterjeschke) for giving feedback on a new N26 dumper config flag that prevents transactions to be imported multiple times [PR #38](https://github.com/schurig/ynab-bank-importer/pull/38)
+* [@peterjeschke](https://github.com/peterjeschke) for fixing a bug that happened when the FinTS username was an integer [PR #35](https://github.com/schurig/ynab-bank-importer/pull/35)
 * [@derintendant](https://github.com/derintendant) for spotting and fixing edge cases [PR #27](https://github.com/schurig/ynab-bank-importer/pull/27) (improves error messages) and [PR #28](https://github.com/schurig/ynab-bank-importer/pull/28) (truncates the payee field if it's too long)
 * [@manuelgrabowski](https://github.com/manuelgrabowski) for implementing a fallback in the FinTS dumper [PR #26](https://github.com/schurig/ynab-bank-importer/pull/26)
 * [@markuspabst](https://github.com/markuspabst) for spotting an error in the readme [PR #11](https://github.com/schurig/ynab-bank-importer/pull/11)

config.sample.yml

@@ -17,4 +17,5 @@ accounts:
     ynab_id: # last hash in the url when you click on the account in YNAB
     username: # n26 username
     password: # n26 password
+    skip_pending_transactions: false # default: false, only imports transactions when they're processed
     set_category: false # default: false, sets the N26 category name as category

docker-compose.dev.yml

@@ -4,5 +4,5 @@ services:
     build:
       context: .
     volumes:
-      - ./config.yml:/usr/app/config.yml
+      - ./:/usr/app/
     command: "ruby /usr/app/run.rb"

lib/dumper.rb

@@ -16,7 +16,7 @@ def self.get_dumper(name)
 
   # rubocop:disable Metrics/MethodLength
   def to_ynab_transaction(transaction)
-    return nil if date(transaction) > Date.today
+    return nil if date(transaction).nil? || date(transaction) > Date.today
     ::TransactionCreator.call(
       account_id: account_id,
       date: date(transaction),

lib/dumper/bbva.rb

@@ -50,7 +50,9 @@ def amount(transaction)
 
     def withdrawal?(transaction)
       text = transaction.description.downcase
-      text.include?('cajero') || text.include?('withdrawal')
+      text.include?('cajero') ||
+        text.include?('withdrawal') ||
+        text.include?('efectivo')
     end
 
     def normalize_iban(iban)

lib/dumper/fints.rb

@@ -7,8 +7,8 @@ class Fints < Dumper
 
     def initialize(params = {})
       @ynab_id  = params.fetch('ynab_id')
-      @username = params.fetch('username')
-      @password = params.fetch('password')
+      @username = params.fetch('username').to_s
+      @password = params.fetch('password').to_s
       @iban     = params.fetch('iban')
       @endpoint = params.fetch('fints_endpoint')
       @blz      = params.fetch('fints_blz')
@@ -32,29 +32,29 @@ def account_id
 
     def date(transaction)
       transaction.entry_date || transaction.date
+    rescue NoMethodError
+      # https://github.com/schurig/ynab-bank-importer/issues/52
+      # Some banks think Feb 29 and 30 exist in non-leap years.
+      entry_date(transaction) || to_date(transaction['date'])
     end
 
     def payee_name(transaction)
-      parse_transaction_at(32, transaction).try(:strip)
+      transaction.name.try(:strip)
     end
 
     def payee_iban(transaction)
-      parse_transaction_at(31, transaction)
+      transaction.iban
     end
 
     def memo(transaction)
-      parse_transaction_at(20, transaction).try(:strip)
+      [
+        transaction.description,
+        transaction.information
+      ].compact.join(' / ').try(:strip)
     end
 
     def amount(transaction)
-      amount =
-        if transaction.funds_code == 'D'
-          "-#{transaction.amount}"
-        else
-          transaction.amount
-        end
-
-      (amount.to_f * 1000).to_i
+      (transaction.amount * transaction.sign * 1000).to_i
     end
 
     def withdrawal?(transaction)
@@ -65,38 +65,45 @@ def withdrawal?(transaction)
     end
 
     def import_id(transaction)
-      data = [transaction_type(transaction),
-              transaction.date,
-              transaction.amount,
-              transaction.funds_code,
-              transaction.reference.try(:downcase),
-              payee_iban(transaction),
-              payee_name(transaction).try(:downcase),
-              @iban].join
-
-      Digest::MD5.hexdigest(data)
+      Digest::MD5.hexdigest(transaction.source)
     end
 
-    def transaction_type(transaction)
-      # Changing the result of this method will
-      # change the hash returned by the `import_id` which
-      # could will result in duplicated entries.
-
-      str = parse_transaction_at(0, transaction).encode('iso-8859-1')
-                                                .force_encoding('utf-8')
-      return nil unless str
-      str[1..-1]
+    # Patches
+
+    # taken from https://github.com/railslove/cmxl/blob/master/lib/cmxl/field.rb
+    # and modified so that it takes the last day of the month if the provided day
+    # doesn't exist in that month.
+    # See issue: https://github.com/schurig/ynab-bank-importer/issues/52
+    DATE = /(?<year>\d{0,2})(?<month>\d{2})(?<day>\d{2})/
+    def to_date(date, year = nil)
+      if match = date.to_s.match(DATE)
+        year ||= "20#{match['year'] || Date.today.strftime('%y')}"
+        month = match['month']
+        day = match['day']
+
+        begin
+          Date.new(year.to_i, month.to_i, day.to_i)
+        rescue ArgumentError
+          # Take the last day of that month
+          Date.civil(year.to_i, month.to_i, -1)
+        end
+      else
+        date
+      end
     end
 
-    def parse_transaction_at(position, transaction)
-      # I don't know who invented this structure but I hope
-      # the responsible people know how inconvenient it is.
+    def entry_date(transaction)
+      data = transaction.data
+      date = to_date(data['date'])
 
-      seperator = transaction.details.seperator
-      array = transaction.details.source.split("#{seperator}#{position}")
-      return nil if array.size < 2
+      return unless transaction.data['entry_date'] && date
 
-      array.last.split(seperator).first
+      entry_date_with_date_year = to_date(data['entry_date'], date.year)
+      if date.month == 1 && date.month < entry_date_with_date_year.month
+        to_date(data['entry_date'], date.year - 1)
+      else
+        to_date(data['entry_date'], date.year)
+      end
     end
   end
 end