Last active
February 8, 2025 04:02
-
-
Save nananek/9ca09288aec1848de38f52189dd40b04 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import pdfplumber | |
| import cv2 | |
| import numpy as np | |
| import csv | |
| import sys | |
| # Use Tkinter for file dialog only on Windows | |
| if sys.platform.startswith("win"): | |
| import tkinter as tk | |
| from tkinter import filedialog | |
| def process_page(page, dpi=300): | |
| # Convert the page to an image at 300 DPI | |
| pil_image = page.to_image(resolution=dpi).original | |
| np_image = np.array(pil_image.convert('L')) | |
| np_image_org = np_image[:, :] | |
| _, np_image = cv2.threshold(np_image, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) | |
| np_image = cv2.bitwise_not(np_image) | |
| for i in range(np_image.shape[0]): | |
| if np.count_nonzero(np_image[i]) > np_image.shape[1] * .7: | |
| np_image[i] = 255 | |
| else: | |
| np_image[i] = 0 | |
| border_size = 3 | |
| for i in range(border_size, np_image.shape[0]): | |
| if np.count_nonzero(np_image[(i - border_size):i, 0]) > 0: | |
| np_image[i] = 0 | |
| borders, = np.nonzero(np_image[:, 0]) | |
| np_words = np.zeros((np_image.shape[0], np_image.shape[1]), dtype=np.uint8) | |
| scale = np_image.shape[1] / page.width | |
| char_list = [] | |
| for char in page.chars: | |
| x0, x1 = char["x0"], char["x1"] | |
| top, bottom = char["top"], char["bottom"] | |
| # 文字の中心座標を計算 | |
| x_center = (x0 + x1) / 2 | |
| y_center = (top + bottom) / 2 | |
| # 拡大・縮小後の幅と高さを計算 | |
| width = (x1 - x0) * 1.7 | |
| height = (bottom - top) * .9 | |
| # 中心を基準に新しい座標を計算 | |
| x0_adj = x_center - width / 2 | |
| x1_adj = x_center + width / 2 | |
| top_adj = y_center - height / 2 | |
| bottom_adj = y_center + height / 2 | |
| # ピクセル座標に変換 | |
| x0_px = int(x0_adj * scale) | |
| x1_px = int(x1_adj * scale) | |
| top_px = int(top_adj * scale) | |
| bottom_px = int(bottom_adj * scale) | |
| # 白い矩形 (255: 白) | |
| cv2.rectangle(np_words, (x0_px, top_px), (x1_px, bottom_px), 255, -1) | |
| char_list.append((char["text"], x_center * scale, y_center * scale)) | |
| char_list = sorted(char_list, key=lambda x: x[1]) | |
| if len(borders) < 1: | |
| return | |
| top_border = borders[0] | |
| texttablelines = [] | |
| for next_border in borders[1:]: | |
| table_line = np_words[top_border:next_border, :] | |
| column_borders = [] | |
| prev = 0 | |
| for i, v in enumerate(np.count_nonzero(table_line, axis=0), 1): | |
| if prev < 1 and v > 0: | |
| column_borders.append(i) | |
| prev = v | |
| if len(column_borders) < 1: | |
| continue | |
| column_borders.append(table_line.shape[1]) | |
| left_column_border = column_borders[0] | |
| textcolumns = [] | |
| for right_column_border in column_borders[1:]: | |
| column = table_line[:, left_column_border:right_column_border] | |
| linebreaks = [] | |
| prev = 0 | |
| for i, v in enumerate(np.count_nonzero(column, axis=1)): | |
| if prev < 1 and v > 0: | |
| linebreaks.append(i) | |
| prev = v | |
| linebreaks.append(column.shape[0]) | |
| top_line_break = linebreaks[0] | |
| textlines = [] | |
| for bottom_line_break in linebreaks[1:]: | |
| top = top_border + top_line_break | |
| left = left_column_border | |
| right = right_column_border | |
| bottom = top_border + bottom_line_break | |
| text = "".join(x[0] for x in char_list if left < x[1] and x[1] < right and top < x[2] and x[2] < bottom).strip() | |
| textlines.append(text) | |
| top_line_break = bottom_line_break | |
| textcolumns.append("\n".join(textlines).strip()) | |
| left_column_border = right_column_border | |
| texttablelines.append(textcolumns) | |
| top_border = next_border | |
| return texttablelines | |
| def save_to_csv(data, filename): | |
| with open(filename, mode='w', newline='\n', encoding='utf-8-sig') as file: | |
| writer = csv.writer(file) | |
| for table in data: | |
| if table: | |
| writer.writerows(table) | |
| writer.writerow([]) # Separate tables by a blank line | |
| def main(): | |
| if sys.platform.startswith("win"): | |
| root = tk.Tk() | |
| root.withdraw() | |
| pdf_path = filedialog.askopenfilename(title="Select PDF File", filetypes=[("PDF Files", "*.pdf")]) | |
| if not pdf_path: | |
| return | |
| csv_path = filedialog.asksaveasfilename(title="Save CSV File", defaultextension=".csv", filetypes=[("CSV Files", "*.csv")]) | |
| if not csv_path: | |
| return | |
| else: | |
| pdf_path = "amex.pdf" # Hardcoded for macOS | |
| csv_path = "output.csv" | |
| tables = [] | |
| with pdfplumber.open(pdf_path) as pdf: | |
| for page in pdf.pages: | |
| tables.append(process_page(page)) | |
| save_to_csv(tables, csv_path) | |
| print(f"Saved extracted table data to {csv_path}") | |
| if __name__ == "__main__": | |
| main() |
We can make this file beautiful and searchable if this error is corrected: It looks like row 3 should actually have 4 columns, instead of 3 in line 2.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ご利用明細,,ご利用金額(外貨),ご利用金額(円) | |
| 1月5日,"Amazon US | |
| 電子機器",1USD = 148.50JPY,"14,850" | |
| 1月8日,"スターバックス 東京都渋谷区 | |
| 飲食","1,892" | |
| 1月12日,"アップルストア 大阪府大阪市 | |
| 電子機器","9,550" | |
| 1月15日,"マクドナルド 愛知県名古屋市 | |
| 飲食","1,623" | |
| 1月18日,"ウーバー 神奈川県横浜市 | |
| 交通","4,935" | |
| 1月22日,Netflix (US) - サブスクリプション,"1,490" | |
| 1月25日,"セブンイレブン 福岡県福岡市 | |
| 食料品","2,250" | |
| 1月28日,Google Play (US) - アプリ,1USD = 148.80JPY,"7,440" | |
| 2月2日,"スターバックス 東京都港区 | |
| 飲食","1,800" | |
| 2月4日,"イトーヨーカドー 千葉県船橋市 | |
| 衣類","3,250" | |
| 2月5日,"ユニクロ 大阪府堺市 | |
| 衣類","6,550" | |
| 2月6日,"ローソン 兵庫県神戸市 | |
| 食料品","1,400" | |
| 2月7日,"カフェ・ド・クリエ 東京都新宿区 | |
| 飲食",950 | |
| 2月9日,"ソフトバンク 東京都千代田区 | |
| 通信","10,000" | |
| 2月10日,"ロフト 京都府京都市 | |
| 雑貨","2,550" | |
| 2月12日,"タワーレコード 神奈川県横浜市 | |
| 音楽","1,700" | |
| 2月14日,"じゃらん 埼玉県さいたま市 | |
| 旅行","8,200" | |
| 2月16日,"ドン・キホーテ 北海道札幌市 | |
| 雑貨","5,300" | |
| 2月17日,"大丸 福岡県北九州市 | |
| 百貨店","4,800" | |
| 2月18日,"三越 愛知県名古屋市 | |
| 百貨店","7,200" | |
| ご利用明細,,ご利用金額(外貨),ご利用金額(円) | |
| 2月19日,"ヨドバシカメラ 東京都新宿区 | |
| 電子機器","12,000" | |
| 2月20日,"銀座三越 東京都中央区 | |
| 百貨店","6,400" | |
| 2月22日,"ビックカメラ 大阪府大阪市 | |
| 電子機器","9,750" | |
| 2月23日,"コストコ 千葉県市川市 | |
| 食料品","15,000" | |
| 2月25日,"イトーヨーカドー 東京都八王子市 | |
| 衣類","7,800" | |
| 2月26日,"セブンイレブン 埼玉県川口市 | |
| 食料品","2,100" | |
| 2月28日,"ドン・キホーテ 東京都足立区 | |
| 雑貨","4,500" | |
| 3月1日,"マツモトキヨシ 福島県郡山市 | |
| 薬局","3,750" | |
| 3月2日,"アップルストア 東京都渋谷区 | |
| 電子機器",1USD = 149.20JPY,"4,460" | |
| 3月3日,"ユニクロ 神奈川県川崎市 | |
| 衣類","5,200" | |
| 3月4日,"マクドナルド 大阪府堺市 | |
| 飲食","1,250" | |
| 3月5日,Amazon (US) - 本,1USD = 148.90JPY,"7,445" | |
| 3月6日,"スターバックス 東京都中央区 | |
| 飲食","2,100" | |
| 3月7日,"セブンイレブン 静岡県浜松市 | |
| 食料品","1,800" | |
| 3月8日,"ユニクロ 東京都豊島区 | |
| 衣類","3,450" | |
| 3月9日,"タワーレコード 東京都渋谷区 | |
| 音楽","2,800" | |
| 3月10日,"ヤマダ電機 群馬県高崎市 | |
| 家電","7,100" | |
| 3月11日,"スリーエフ 東京都八王子市 | |
| 食料品",950 | |
| 3月12日,"ローソン 大阪府大阪市 | |
| 食料品","1,200" | |
| 3月13日,"コストコ 神奈川県川崎市 | |
| 食料品","18,500" | |
| ご利用明細,,ご利用金額(外貨),ご利用金額(円) | |
| 3月14日,"ビックカメラ 東京都立川市 | |
| 電子機器","6,600" | |
| 3月15日,"ドン・キホーテ 東京都池袋区 | |
| 雑貨","5,300" | |
| 3月16日,"ローソン 茨城県水戸市 | |
| 食料品","2,500" | |
| 3月17日,"マクドナルド 兵庫県神戸市 | |
| 飲食","1,150" | |
| 3月18日,"三越 東京都千代田区 | |
| 百貨店","9,200" | |
| 3月19日,"ビックカメラ 埼玉県さいたま市 | |
| 電子機器","7,700" | |
| 3月20日,"ヨドバシカメラ 大阪府高槻市 | |
| 電子機器","8,900" | |
| 3月21日,"セブンイレブン 東京都新宿区 | |
| 食料品","1,500" | |
| 3月22日,Amazon (US) - 家具,1USD = 149.00JPY,"7,450" | |
| 3月23日,"大丸 東京都渋谷区 | |
| 百貨店","7,200" | |
| 3月24日,"ユニクロ 大阪府堺市 | |
| 衣類","4,650" | |
| 3月25日,"ソフトバンク 東京都新宿区 | |
| 通信","9,800" | |
| 3月26日,"ローソン 東京都渋谷区 | |
| 食料品","1,750" | |
| 3月27日,"ジョーシン 大阪府大阪市 | |
| 家電","5,350" | |
| 3月28日,"イトーヨーカドー 東京都新宿区 | |
| 衣類","10,250" | |
| 3月29日,"スターバックス 東京都渋谷区 | |
| 飲食","2,700" | |
| 3月30日,"マクドナルド 東京都世田谷区 | |
| 飲食","1,500" | |
| 3月31日,"ビックカメラ 東京都新宿区 | |
| 電子機器","9,300" | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment