Modification en profondeur du facebook parser

et enrichissement des classes modèle

+ ajout d'un dossier output dans le gitignore

.gitignore

@@ -1,5 +1,7 @@
 *.pyc
 
+output/*
+
 # Packages
 *.egg
 !/tests/**/*.egg

million/model/fb_export.py

@@ -1,11 +1,23 @@
-
-from typing import List
+from typing import Any, List
 from pydantic import BaseModel
-from million.model.message import Message
 
+from million.model.message import Message
 from million.model.participant import Participant
 
+class Image(BaseModel):
+    creation_timestamp: int
+    uri: str
+
+class JoinableMode(BaseModel):
+    mode: int
+    link: str
 
 class FacebookExport(BaseModel):
     messages: List[Message]
-    participants: List[Participant]
+    participants: List[Participant]
+    title: str
+    is_still_participant: bool
+    thread_path: str
+    magic_words: List[Any]
+    image: Image
+    joinable_mode: JoinableMode

million/model/message.py

@@ -1,14 +1,54 @@
-
+from datetime import datetime
 from math import floor
-from typing import Optional
+from typing import Any, List, Optional
 from pydantic import BaseModel
 
+class Reaction(BaseModel):
+    reaction: str
+    actor: str
+
+class AudioFile(BaseModel):
+    uri: str
+    creation_timestamp: int
+
+class Video(BaseModel):
+    uri: str
+    creation_timestamp: int
+
+class Photo(BaseModel):
+    uri: str
+    creation_timestamp: int
+
+class Gif(BaseModel):
+    uri: str
+
+class Share(BaseModel):
+    link: str
+    share_text: str
+
+class Sticker(BaseModel):
+    uri: str
+    ai_stickers: List[Any]
 
 class Message(BaseModel):
     sender_name: str
     timestamp_ms: int
-    content: Optional[str] = None
-    is_geoblocked_for_viewer: Optional[bool] = None
+    content: str | None = None
+    sticker: Sticker | None = None
+    share: Share | None = None
+    photos: List[Photo] | None = None
+    videos: List[Video] | None = None
+    gifs: List[Gif] | None = None
+    audio_files: List[AudioFile] | None = None
+    call_duration: int | None = None
+    reactions: List[Reaction] | None = None
+    is_unsent: bool | None = None
+    is_geoblocked_for_viewer: bool
+
+    def __str__(self) -> str:
+        dt = datetime.fromtimestamp(self.timestamp_ms / 1000)
+        dt_str = dt.strftime("%d/%m/%Y, %H:%M:%S")
+        return f"{self.sender_name}({dt_str}) : {self.content}"
 
     def get_counted_value(self):
         """

million/model/participant.py

@@ -3,4 +3,7 @@ from pydantic import BaseModel
 
 
 class Participant(BaseModel):
-    name: str
+    name: str
+
+    def __hash__(self):
+        return hash(self.name)

million/parse/fb_exports.py

@@ -1,37 +1,43 @@
 
-import json
-import os
-import re
+import json, os, re
 from typing import List
 
 from million.model.fb_export import FacebookExport
-from million.model.message import Message
-
-
-class FacebookExportParser:
-
-    def __init__(self):
-        pass
-
-    def parse(self, file_dir) -> FacebookExport:
-        files = [file_dir +
-                 f for f in os.listdir(file_dir) if f.endswith('.json')]
-        messages = []
-        participants = []
-        for file in files:
-            print(file)
-            with open(file, 'rb') as f:
-                json_data = self.__read_broken_fb_json(f.read())
-                messages += [Message(**m) for m in json_data['messages']]
-                participants += json_data['participants']
-
-        messages.sort(key=lambda m: m.timestamp_ms)
-        return FacebookExport(messages=messages, participants=participants)
-
-    def __read_broken_fb_json(self, binary_data):
-        repaired = re.sub(
-            rb'\\u00([\da-f]{2})',
-            lambda m: bytes.fromhex(m.group(1).decode()),
-            binary_data
-        )
-        return json.loads(repaired.decode('utf8'))
+
+
+def is_file_valid(file_name: str) -> bool:
+    return os.path.splitext(file_name)[-1].lower() == '.json'
+
+def valid_dirfiles(file_dir: str) -> List[str]:
+    return [os.path.join(file_dir, file_name)
+            for file_name in os.listdir(file_dir)
+            if is_file_valid(file_name)]
+
+def parse_file(file_name: str) -> FacebookExport:
+    if not is_file_valid(file_name): return None
+
+    with open(file_name, 'rb') as f:
+        fixed_json = __read_broken_fb_json(f.read())
+        json_data = json.loads(fixed_json)
+        return (FacebookExport(**json_data))
+
+def parse_dirfiles(file_dir: str) -> FacebookExport:
+    exports = [parse_file(f) for f in valid_dirfiles(file_dir)]
+    if len(exports) == 0: return
+
+    for other in exports[1:]:
+        exports[0].messages.extend(other.messages)
+        exports[0].participants.extend(other.participants)
+
+    exports[0].messages.sort(key = lambda m: m.timestamp_ms)
+    exports[0].participants = set(exports[0].participants)
+    return exports[0]
+
+def __read_broken_fb_json(binary_data):
+    repaired = re.sub(
+        rb'\\u00([\da-f]{2})',
+        lambda m: bytes.fromhex(m.group(1).decode()),
+        binary_data
+    )
+
+    return repaired.decode('utf8')