2023-02-16 23:24:22 +02:00
#!/usr/bin/python
"""
2023-02-17 09:16:19 +02:00
Creates receipts in TAMI ' s accounting provider, from bank-reports.
input : ( XXXXX ) bank ' s transactions list, in Excel format [currently: TSV from google-sheets]
output : a new donation - recepit ( type #405) at GreenInvoice.co.il
2023-02-16 23:24:22 +02:00
usage :
2023-02-17 09:16:19 +02:00
bank2invoice . py real < excel - file - 1 > [ < excel - file - 2 > . . . ]
bank2invoice . py search # get latest existing receipts
bank2invoice . py test #
2023-02-16 23:24:22 +02:00
2023-02-17 09:16:19 +02:00
* REAL vs TEST modes : *
by default , running in TEST mode ( inside a sandbox ) .
if you want to get things into the real accounting system , add the command line argument : real
2023-02-16 23:24:22 +02:00
2023-02-17 09:16:19 +02:00
The " test " command is using built - in data file , which is embedded in this module .
2023-02-16 23:24:22 +02:00
started by shaharr . info ,
on request of yair , 2023 - 02 - 15
"""
import os , re , sys , time
from os . path import expanduser , expandvars , isdir , isfile , join , basename , abspath , splitext , getsize , dirname
from os import mkdir , chdir , rename , unlink , walk , popen
from json import dumps , loads
from glob import iglob
from time import sleep , time as currtime
from datetime import timedelta , datetime
import logging
import requests
import tempfile
import green_invoice
from green_invoice . models import (
Currency ,
DocumentLanguage ,
DocumentType ,
PaymentCardType ,
PaymentDealType ,
PaymentType ,
IncomeVatType ,
)
from green_invoice . resources import DocumentResource
class AttrDict ( dict ) :
""" acces dictionary data (dic[ ' key ' ]) as objects (dic.key) """
def __init__ ( self , * args , * * kwargs ) :
super ( AttrDict , self ) . __init__ ( * args , * * kwargs )
self . __dict__ = self
2023-02-17 09:16:19 +02:00
""" # ----- copied from green_invoice/resources/resource.py, merely for documentation! -----
2023-02-16 23:24:22 +02:00
class DocumentType ( enum . IntEnum ) :
PRICE_QUOTE = 10
ORDER = 100
DELIVERY_NOTE = 200
RETURN_DELIVERY_NOTE = 210
TRANSACTION_ACCOUNT = 300
TAX_INVOICE = 305
TAX_INVOICE_RECEIPT = 320
REFUND = 330
RECEIPT = 400
RECEIPT_FOR_DONATION = 405
PURCHASE_ORDER = 500
RECEIPT_OF_A_DEPOSIT = 600
WITHDRAWAL_OF_DEPOSIT = 610
class DocumentStatus ( enum . IntEnum ) :
OPENED_DOCUMENT = 0
CLOSED_DOCUMENT = 1
MANUALLY_MARKED_AS_CLOSED = 2
CANCELING_OTHER_DOCUMENT = 3
CANCELED_DOCUMENT = 4
class DocumentLanguage ( str , enum . Enum ) :
HEBREW = " he "
ENGLISH = " en "
class Currency ( str , enum . Enum ) :
ILS = " ILS "
USD = " USD "
EUR = " EUR "
GBP = " GBP "
class PaymentType ( enum . IntEnum ) :
UNPAID = - 1
DEDUCTION_AT_SOURCE = 0
CASH = 1
CHECK = 2
CREDIT_CARD = 3
ELECTRONIC_FUND_TRANSFER = 4
PAYPAL = 5
PAYMENT_APP = 10
OTHER = 11
2023-02-17 09:16:19 +02:00
# ----- end of documentation ---- """
2023-02-16 23:24:22 +02:00
# ------ CONSTs -------
flags = re . M + re . I + re . S
conffile = expanduser ( ' ~/.bank2invoice.ini ' )
#conf = Config(json_file=expanduser('~/.config/someapp.json'))
2023-02-17 09:16:19 +02:00
payment_type_map = { # see docstring in guess_payment_type()
2023-02-16 23:24:22 +02:00
' ^bit העברת כספים$ ' : 10 ,
' .*מזומן.* ' : 1 ,
}
INITIAL_LATEST_PAYMENT = ' 0000-00-00 '
default_payment_type = 4 # bank wire
default_doctype = DocumentType . RECEIPT_FOR_DONATION
2023-02-17 09:16:19 +02:00
MAX_DAYS = 49
2023-02-16 23:24:22 +02:00
BACK_DAYS = 40
banks = {
4 : " בנק יהב " ,
6 : " בנק מזרחי טפחות " ,
9 : " בנק הדואר " ,
10 : " בנק לאומי " ,
11 : " בנק דיסקונט " ,
12 : " בנק הפועלים " ,
13 : " בנק איגוד " ,
14 : " בנק אוצר החייל " ,
17 : " בנק מרכנתיל דיסקונט " ,
20 : " בנק מזרחי טפחות " ,
22 : " בנק CitiBank " ,
23 : " בנק HSBC " ,
25 : " בנק BNP Paribas " ,
26 : " בנק יובנק " ,
27 : " Barclays Bank PLC " ,
31 : " בנק בינלאומי " ,
34 : " בנק ערבי ישראלי " ,
39 : " בנק SBI State of India " ,
43 : " ג ' ורדן נשיונל בנק PLC.עמאן " ,
46 : " בנק מסד " ,
48 : " בנק קופת העובד הלאומי " ,
50 : ' מרכז סליקה בנקאי (מס " ב) ' ,
52 : " בנק פועלי אגודת ישראל " ,
54 : " בנק ירושלים " ,
59 : ' שב " א ' ,
65 : " חסך קופת חסכון לחינוך " ,
66 : " קהיר-עומאן " ,
67 : " בנק Arab Land " ,
68 : " בנק אוצר השלטון המקומי " ,
68 : " בנק דקסיה " ,
71 : " קומרשיאל ג ' ורדן " ,
73 : " ערב איסמליק " ,
74 : " בריטיש בנק אוף מידל איסט " ,
76 : " השקעות פלשתין " ,
77 : " בנק לאומי למשכנתאות " ,
82 : " אל-קודס לפיתוח והשקעות " ,
83 : " יוניון בנק " ,
84 : " האוזינג " ,
89 : " מספר בנק פלסטין " ,
90 : " בנק דיסקונט למשכנתאות " ,
93 : " ג ' ורדן כווית " ,
99 : " בנק ישראל " ,
}
def err ( msg , fatal = False ) :
msg = msg . strip ( )
if msg [ : 5 ] . lower ( ) != ' error ' :
msg = ' ERROR: ' + msg
sys . stderr . write ( msg + ' \n ' )
if fatal :
sys . exit ( )
def guess_payment_type ( payment , conf ) :
2023-02-17 09:16:19 +02:00
""" for guessing trx classification based on payment ' s comments. """
2023-02-16 23:24:22 +02:00
for key in payment_type_map . keys ( ) :
if re . match ( key , payment . comments ) :
return payment_type_map [ key ]
return default_payment_type
def read_conf ( ) :
conf = AttrDict ( )
if not isfile ( conffile ) :
err ( f ' config file (with top secret data!!) is not found in { conffile } ! ' , True )
for line in open ( conffile ) . read ( ) . splitlines ( ) :
if ' = ' in line and line [ 0 ] != ' # ' :
k , v = line . split ( ' = ' , 2 )
conf [ k . strip ( ) ] = v . strip ( )
return conf
def normalize_dates ( s ) :
2023-02-17 09:16:19 +02:00
""" convert dd/mm/yyyy (eu dates) to yyyy-mm-dd (ISO dates) """
2023-02-16 23:24:22 +02:00
return re . sub ( r ' ([0-3] \ d)/([01] \ d)/(20[23] \ d) ' , r ' \ 3- \ 2- \ 1 ' , s , flags = flags )
def get_existing_documents ( dtype ) :
""" purpose: to prevent duplicates,
and to get last receipt date ( cant add earlier reciept ) """
2023-02-17 09:16:19 +02:00
# search documents since 100 days ago. earlier dates aren't relevant. I hope.
2023-02-16 23:24:22 +02:00
from_date = ( datetime . today ( ) - timedelta ( days = 100 ) ) . strftime ( ' % Y- % m- %d ' )
to_date = iso_date ( )
documentResource = DocumentResource ( )
Payment . latest = INITIAL_LATEST_PAYMENT
results = [ ]
docs = set ( )
2023-02-17 09:16:19 +02:00
PAGES = 6 # just a random number; green-invoice's paging system is beyond logic. 6 sounds nice. 10 was taking too long. 1 page with pagesize 300 was giving only 25 recs per page anyway.
2023-02-16 23:24:22 +02:00
for page in range ( PAGES ) :
print ( f ' downloading existing docs, page { page } / { PAGES } ' )
params = dumps ( {
" page " : page , " pageSize " : 50 ,
" type " : [ dtype ] , " sort " : " documentDate " ,
" fromDate " : from_date , " toDate " : to_date ,
} )
page_results = documentResource . search_document ( params )
page_results = page_results [ ' items ' ]
if not page_results : break
print ( f ' { len ( page_results ) =} ' )
results . extend ( page_results )
for ret in results :
2023-02-17 09:16:19 +02:00
# convert green-invoices entity to our Payment() object. We need that to compare -> prevent duplicates.
2023-02-16 23:24:22 +02:00
doc = Payment ( * * dict (
type = ret [ ' type ' ] ,
pay_date = ret [ ' payment ' ] [ 0 ] [ " date " ] ,
document_date = ret [ ' documentDate ' ] ,
amount = ret [ ' amount ' ] ,
client_name = ret [ ' client ' ] [ ' name ' ] ,
comments = ret [ ' remarks ' ]
) )
2023-02-17 09:16:19 +02:00
docs . add ( doc ) # unique items only
2023-02-16 23:24:22 +02:00
Payment . latest = max ( Payment . latest , doc . pay_date )
print ( f ' { Payment . latest =} vs. { doc . pay_date =} ' )
return docs
def payment_exists ( payment , existing ) :
for doc in existing :
print ( f ' { payment . comments =} >>> { doc . comments =} ' )
if doc == payment :
return True
return False
def main ( xl_file , conf ) :
green_invoice . client . configure (
2023-02-17 09:16:19 +02:00
env = " sandbox " , # actually i didn't test it yet in the real system. donno whats this.
2023-02-16 23:24:22 +02:00
api_key_id = conf . api_key_id ,
api_key_secret = conf . api_key_secret ,
logger = logging . root ,
)
2023-02-17 09:16:19 +02:00
existing = get_existing_documents ( default_doctype ) # for preventing duplicates, later
2023-02-16 23:24:22 +02:00
s = open ( xl_file ) . read ( )
2023-02-17 09:16:19 +02:00
s = normalize_dates ( s ) # convert israeli (european) dates to ISO dates:
2023-02-16 23:24:22 +02:00
# green-invoice requires them,
# plus we must sort by date.
lines = s . splitlines ( )
lines . sort ( )
for line in lines :
2023-02-17 09:16:19 +02:00
if not ' \t ' in line : continue # empty lines
if line [ : 2 ] != ' 20 ' : continue # must start with the year; otherwise it's a header line
2023-02-16 23:24:22 +02:00
payment = Payment ( line = line )
if not payment : continue
if payment_exists ( payment , existing ) :
2023-02-17 09:16:19 +02:00
print ( ' payment already in the system. skip ' )
2023-02-16 23:24:22 +02:00
continue
id , url = create_receipt ( payment , conf )
2023-02-17 09:16:19 +02:00
# @todo: send url to donnor. but we dont have their contact here.
2023-02-16 23:24:22 +02:00
def create_receipt ( payment , conf ) :
documentResource = DocumentResource ( )
payment_type = PaymentType . ELECTRONIC_FUND_TRANSFER
today = datetime . today ( )
doc_date = max ( Payment . latest , payment . pay_date ) # latest from payment vs. latest receipt
days = ( today - datetime . fromisoformat ( doc_date ) ) . days # how many days ago is it?
if days > = MAX_DAYS : # if too many days,
doc_date = ( today - timedelta ( days = BACK_DAYS ) ) . strftime ( ' % Y- % m- %d ' ) # set to 40 days ago
print ( f ' { doc_date =} | { payment . pay_date =} | { Payment . latest =} | { iso_date ( ) =} ' )
doc = documentResource . create (
{
" type " : default_doctype , # see types above
" client " : { " name " : payment . client_name , " add " : False , } ,
" currency " : payment . currency ,
" lang " : DocumentLanguage . HEBREW ,
" date " : doc_date ,
" signed " : True ,
" rounding " : False ,
" remarks " : payment . comments ,
2023-02-17 09:16:19 +02:00
#"income": [ # ------- according to support, this is for חשבונית מס which we don't use in our non-profit org.
2023-02-16 23:24:22 +02:00
# {
# "price": payment.amount,
# "currency": payment.currency,
# "quantity": 1,
# "description": DEFAULT_TXN_DESCRIPTION,
# "vatType": IncomeVatType.DEFAULT, # based on the business type
# }
#],
2023-02-17 09:16:19 +02:00
" payment " : [ # ---- according to support, this is for קבלה, which is the specific accounting status we use here.
2023-02-16 23:24:22 +02:00
{
" type " : guess_payment_type ( payment , conf ) ,
" date " : payment . pay_date ,
" dealType " : PaymentDealType . REGULAR ,
#"cardNum": "4242",
#"cardType": PaymentCardType.VISA,
" bankName " : banks . get ( payment . bank , 0 ) ,
" bankBranch " : str ( payment . snif ) ,
" bankAccount " : str ( payment . account ) ,
" price " : payment . amount ,
" currency " : payment . currency ,
}
] ,
}
)
id = doc [ ' id ' ]
url = doc [ ' id ' ]
return id , url
class Payment :
2023-02-17 09:16:19 +02:00
""" represents a transaction ' s data """
latest = INITIAL_LATEST_PAYMENT # global scope
2023-02-16 23:24:22 +02:00
def __init__ ( self , * * kw ) :
""" parse a line from the bank report;
right now i ' ve downloaded a tab-separated file from google-spreadsheet;
2023-02-17 09:16:19 +02:00
2023-02-16 23:24:22 +02:00
overloaded usage :
obj = Payment ( line ) < - string , tab - separated , from excel
obj = Payment ( { date : x , name : y , . . . } ) < - all object properties
"""
self . type = default_doctype
if ' line ' in kw :
2023-02-17 09:16:19 +02:00
parts = kw [ ' line ' ] . split ( ' \t ' )
2023-02-16 23:24:22 +02:00
if len ( parts ) != 7 :
print ( ' shit, bank data record must have 7 columns exactly ' )
raise Exception # either a bug, or format change, must re-adapt the code !
self . pay_date , self . client_name , self . bank , self . snif , self . account , amount , self . comments = parts
#self.pay_date = f'{d[6:10]}-{d[3:5]}-{d[0:2]}' # european date to ISO
self . amount = float ( amount )
self . currency = Currency . ILS # default?!??! assumption...
else :
for k , v in kw . items ( ) :
setattr ( self , k , v )
def __eq__ ( self , other ) :
2023-02-17 09:16:19 +02:00
""" represents the unique signature for transactions;
used for comparing against existing records / transactions : ( pay1 == pay2 )
we get from the bank these items :
pay_date , client_name , bank , snif , account #, amount, comments """
2023-02-16 23:24:22 +02:00
return ( self . type == other . type
and self . pay_date == other . pay_date
and self . amount == other . amount
and self . client_name == other . client_name
and self . comments == other . comments ) # this is tricky, @todo
def __hash__ ( self ) :
2023-02-17 09:16:19 +02:00
""" makes this class hashable; i ' m using it to use within set() """
2023-02-16 23:24:22 +02:00
return hash ( f ' { self . pay_date } { self . amount } { self . client_name } ' )
def iso_date ( ) :
2023-02-17 09:16:19 +02:00
""" todays date as ISO """
2023-02-16 23:24:22 +02:00
return datetime . today ( ) . strftime ( ' % Y- % m- %d ' )
2023-02-17 09:16:19 +02:00
2023-02-16 23:24:22 +02:00
def self_test ( ) :
conf = read_conf ( )
conf . test = True
conf . api_key_id = conf . sanbox_api_id
conf . api_key_secret = conf . sandbox_api_secret
conf . api_url = conf . sandbox_api_url
f = tempfile . mktemp ( prefix = ' tami-kabala-test ' , suffix = ' .test ' )
test_data = '''
# this is a comment
תאריך רישום שם בנק סניף חשבון סכום הערות
29 / 12 / 2022 גגג גוגייג 10 123 3434343 200.00 במעטפות שמנות
29 / 12 / 2022 צגי גגוג גוגג 43 234 0 30.00 סדנת עיזים
21 / 12 / 2022 Amd גייג 12 345 0 50.50 bit העברת כספים
19 / 12 / 2022 גגיג גוגי 17 456 12345678 300.00
18 / 12 / 2022 גגג גגיגיג 20 567 987654 333.00 US PERSON תרומות
18 / 12 / 2026 גגג גגיגיג 20 567 987654 333.00 תאריך חדש מדי
18 / 12 / 2021 גגג גגגיג 34 567 987654 222.00 תאריך ישן מדי
'''
open ( f , ' w ' ) . write ( test_data )
main ( f , conf )
2023-02-17 09:16:19 +02:00
assert True # @todo; needs much better testing;
2023-02-16 23:24:22 +02:00
conf = read_conf ( )
2023-02-17 09:16:19 +02:00
if f ' { sys . version_info . major : 02d } { sys . version_info . minor : 02d } ' < ' 0308 ' :
msg = f ' WARNING !!! ' * 5 + f " \n ' bank2invoice.py ' was tested on python 3.10, not on { sys . version_info . major } . { sys . version_info . minor } . OTOH yair wants to run it on py3.8, Let ' s see if it really does! "
2023-02-16 23:24:22 +02:00
print ( msg )
2023-02-17 09:16:19 +02:00
err ( msg ) # just a non blocking, non-fatal warning
2023-02-16 23:24:22 +02:00
if __name__ == " __main__ " :
args = sys . argv [ 1 : ]
conf . test = ' real ' not in args
conf . writing_date = None # meaning,
if conf . test :
conf . api_key_id = conf . sanbox_api_id
conf . api_key_secret = conf . sandbox_api_secret
conf . api_url = conf . sandbox_api_url
print ( ' test mode ' )
files = set ( )
2023-02-17 09:16:19 +02:00
for arg in args : # parsing command line without external tricks
2023-02-16 23:24:22 +02:00
if ' search ' in args :
2023-02-17 09:16:19 +02:00
pass # @todo; make it print existing transactions; No use for that now.
2023-02-16 23:24:22 +02:00
elif ' test ' in args :
self_test ( )
exit ( )
2023-02-17 09:16:19 +02:00
elif arg [ : 6 ] == ' --date ' : # in case we want to register recepits as a certain date; not implemented(?)
2023-02-16 23:24:22 +02:00
d = arg [ 7 : ]
today = iso_date ( )
if not re . match ( r ' ^20[23] \ d-[01] \ d-[0-3] \ d$ ' , d ) or arg < = today :
err ( ' bad writing date, or writing date in the future ' , True )
else :
conf . writing_date = arg
elif isfile ( arg ) :
files . add ( arg )
for f in files :
main ( f , conf )
if not files : # if no data to process
print ( __doc__ )
err ( ' warning: no bank files provided ' , True )
print ( f ' { c } records added ' )