from tika import parser
import os
import glob
def parse_value(s):
sign = 1
if s[0] == '(' and s[-1] == ')':
sign = -1
s = s[1:-1]
return sign * int("".join(s.split(",")))
def parse_line(line): #parse line of a table.
cols = [c for c in line.strip().split(" ") if c != ""]
if len(cols) < 2:
return None, None
try:
return " ".join(cols[:-1]), parse_value(cols[-1])
except ValueError:
return None, None
def parse_dividend_tax(lines):
try:
div_tax_line = [l for l in lines if l.startswith("1 Dividends")][0]
div_tax_line = div_tax_line.replace(r",000.","") #remove end of line and thousands sign
return parse_value(div_tax_line.split("$")[1])
except IndexError:
return 0 #For VTI, there is no dividend tax payed by the fund (aka ETF)
def parse_page(page):
mapping = dict()
lines = page.split("\n")
for l in lines:
k,v = parse_line(l)
if k is not None:
mapping[k] = v
mapping["Dividend tax"] = parse_dividend_tax(lines)
if "Dividends" not in mapping.keys():
mapping["Dividends"] = mapping[[m for m in mapping.keys() if m.startswith("Dividends")][0]] # get rid of footnote markers.
mapping["Dividends net"] = mapping["Dividends"]
mapping["Dividents received"] = mapping["Dividends net"] + mapping["Dividend tax"]
mapping["Dividend tax rate"] = mapping["Dividend tax"]/mapping["Dividents received"]
mapping["Name"] = [l for l in lines if l.endswith("Index Fund")][0]
return mapping
def parse_net_value_of_page(page):
lines_raw = page.split("\n")
end_of_period_net_value = [l for l in lines_raw if l.startswith("End of Period")][0].replace("End of Period", "").strip().split(" ")
return {
"Name" : [l for l in lines_raw if l.endswith("Index Fund")][0],
"End of period": parse_value(end_of_period_net_value[0]),
"Begin of period": parse_value(end_of_period_net_value[1]) # end of previous year equals to current value this year.
}
def parse_file(filename):
raw = parser.from_file(filename)
content = raw['content']
pages_raw = content.split("\n\n\n\n")
#Step 1: Statement of Operations table
div_pages = []
for i in range(len(pages_raw)):
p = pages_raw[i]
if "Investment Income" in p and "Dividends" in p and "Securities Lending—Net" in p:
if "Net Increase (Decrease) in Net Assets Resulting from Operations" in p:
div_pages.append(p)
else:
# the Statement of Operations spans two tables.
assert "Net Increase (Decrease) in Net Assets Resulting from Operations" in pages_raw[i+1]
div_pages.append(p + pages_raw[i+1])
dividends = [parse_page(p) for p in div_pages]
# Step 2: Statement of Changes in Net Assets table
net_assets_pages = [p for p in pages_raw if "End of Period" in p and "Increase (Decrease) in Net Assets" in p]
net_values = [parse_net_value_of_page(p) for p in net_assets_pages]
assert len(dividends) == len(net_values)
for div,net in zip(dividends, net_values):
assert div["Name"] == net["Name"]
for k,v in net.items():
div[k] = v
return dividends