Des utilitaires pour filtrer les messages et classer les participants

million/analyze/filter.py

@@ -1,40 +0,0 @@
-from __future__ import annotations
-from datetime import datetime
-from typing import List
-from million.model.filter.MessageFilter import MessageFilter
-from million.model.message import Message
-
-
-def before(message: Message, date: datetime):
-    return message.date < date
-
-
-def after(message: Message, date: datetime):
-    return message.date > date
-
-
-def contains(message: Message, content: str):
-    return message.content is not None and content.lower() in message.content.lower()
-
-
-def is_number(message: Message, is_number: bool):
-    return message.content.isdigit() == is_number
-
-
-def filter(messages: List[Message], criterias: MessageFilter):
-    print(criterias)
-    conditions = [
-        [before, criterias.before] if criterias.before else None,
-        [after, criterias.after] if criterias.after else None,
-        [contains, criterias.contains] if criterias.contains else None,
-        [is_number, criterias.is_number] if criterias.is_number else None
-    ]
-
-    return [
-        message for message in messages
-        if all(
-            condition[0](message, condition[1])
-            for condition in conditions
-            if condition
-        )
-    ]

million/analyze/filter/message.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+from datetime import datetime
+import functools
+from typing import Any, Callable, List
+from million.analyze.retain_counts import is_count
+from million.model.filter.MessageFilter import MessageFilter, MessageOrder
+from million.model.message import Message
+
+
+def before(message: Message, date: datetime):
+    return message.date < date
+
+
+def after(message: Message, date: datetime):
+    return message.date > date
+
+
+def contains(message: Message, content: str):
+    return message.content is not None and content.lower() in message.content.lower()
+
+
+def is_number(message: Message, is_number: bool):
+    return is_count(message) == is_number
+
+
+def sender_name(message: Message, sender_name: str):
+    return message.sender_name == sender_name
+
+
+def having_reactions(message: Message, having_reactions: bool):
+    return bool(message.reactions) == having_reactions
+
+
+def list_to_tuple(function: Callable) -> Any:
+    """Custom decorator function, to convert list to a tuple."""
+
+    def wrapper(*args, **kwargs) -> Any:
+        args = tuple(tuple(x) if isinstance(x, list) else x for x in args)
+        kwargs = {k: tuple(v) if isinstance(v, list)
+                  else v for k, v in kwargs.items()}
+        result = function(*args, **kwargs)
+        result = tuple(result) if isinstance(result, list) else result
+        return result
+
+    return wrapper
+
+
+def order_reaction(messages: List[Message]):
+    return sorted(
+        messages,
+        key=lambda x: len(x.reactions) if x.reactions else 0,
+        reverse=True)
+
+
+def order_date(messages: List[Message]):
+    return sorted(messages, key=lambda x: x.date, reverse=True)
+
+
+@list_to_tuple
+@functools.lru_cache(maxsize=10_000)
+def filter(messages: List[Message], criterias: MessageFilter = None, order: MessageOrder = None, limit: int = None) -> List[Message]:
+    print(criterias)
+    conditions = [
+        [sender_name, criterias.sender_name] if criterias.sender_name else None,
+        [after, criterias.after] if criterias.after else None,
+        [before, criterias.before] if criterias.before else None,
+        [contains, criterias.contains] if criterias.contains else None,
+        [is_number, criterias.is_number] if criterias.is_number is not None else None,
+        [having_reactions, criterias.having_reactions] if criterias.having_reactions is not None else None,
+    ]
+
+    filtered = [
+        message for message in messages
+        if all(
+            condition[0](message, condition[1])
+            for condition in conditions
+            if condition
+        )
+    ]
+
+    if order == MessageOrder.reactions:
+        filtered = order_reaction(filtered)
+    elif order == MessageOrder.date:
+        filtered = order_date(filtered)
+
+    if limit:
+        filtered = filtered[:limit]
+
+    return filtered
+
+
+def count(messages: List[Message], criterias: MessageFilter):
+    return len(filter(messages, criterias))

million/analyze/filter/participants.py

@@ -0,0 +1,46 @@
+
+
+from typing import List
+
+from pydantic import BaseModel
+from million.model.filter.MessageFilter import MessageFilter
+from million.model.filter.ParticipantFilter import OrderDirection, ParticipantFilter
+from million.model.message import Message
+from million.model.participant import Participant
+from million.analyze.filter.message import filter as message_filter
+
+
+class ParticipantCount(BaseModel):
+    participant: Participant
+    count: int
+
+
+def order_count(participants: List[Participant], messages: List[Message], criterias: MessageFilter = None, direction: OrderDirection = OrderDirection.desc) -> ParticipantCount:
+    result = [
+        ParticipantCount(
+            participant=participant,
+            count=len(
+                message_filter(
+                    messages,
+                    criterias=MessageFilter(**{
+                        **(criterias.model_dump() if criterias else dict()),
+                        "sender_name": participant.name
+                    })
+                )
+            ))
+
+        for participant in participants
+    ]
+    result.sort(key=lambda p: p.count, reverse=True)
+    return [result for result in result]
+
+
+def filter(messages: List[Message], participants: List[Participant], criterias: ParticipantFilter) -> List[ParticipantCount]:
+    result = [
+        participant for participant in participants
+    ]
+
+    result = order_count(
+        result, messages, criterias=criterias, direction=criterias.order_dir)
+
+    return result

million/analyze/retain_counts.py

@@ -4,12 +4,18 @@ from typing import List
 from million.model.message import Message
 
 
-def retain_counts(messages : List[Message])-> List[Message]:
+def is_count(message: Message) -> bool:
+    """
+    Check if the message is a count
+    """
+    return message.content is not None and bool(re.search('(\d{2,}|^\d$)', message.content))
+
+
+def retain_counts(messages: List[Message]) -> List[Message]:
     """
     Retain only the messages that have a content
     """
     return [
-        m for m in messages 
-        if m.content and
-        re.search('(\d{2,}|^\d$)', m.content)
-        ]
+        m for m in messages
+        if is_count(m)
+    ]

million/http/app.py

@@ -1,8 +1,11 @@
 from typing import Annotated, List
 from fastapi import FastAPI, Query
-from million.model.filter.MessageFilter import MessageFilter
+from million.model.filter.MessageFilter import MessageFilter, MessageOrder
+from million.model.filter.ParticipantFilter import ParticipantFilter
 from million.model.message import Message
-from million.analyze.filter import filter
+from million.analyze.filter.message import filter as message_filter
+from million.analyze.filter.participants import ParticipantCount, filter as participants_filter
+from million.model.participant import Participant
 import million.parse.fb_exports as fb
 
 
@@ -13,6 +16,18 @@ export = fb.parse_dirfiles(DATA_PATH)
 app = FastAPI()
 
 
+class GetMessageForm(MessageFilter):
+    order: MessageOrder = None
+    limit: int = 100
+
+
 @app.get("/messages")
-def read_messages(form: Annotated[MessageFilter, Query()]) -> List[Message]:
-    return filter(export.messages, form)
+def read_messages(
+    form: Annotated[GetMessageForm, Query()]
+) -> List[Message]:
+    return message_filter(export.messages, form, form.order, form.limit)
+
+
+@app.get("/participants")
+def read_participants(form: Annotated[ParticipantFilter, Query()]) -> List[ParticipantCount]:
+    return participants_filter(export.messages, export.participants, form)

million/model/filter/MessageFilter.py

@@ -1,9 +1,20 @@
 from datetime import datetime
+from enum import Enum
 from pydantic import BaseModel
 
 
+class MessageOrder(str, Enum):
+    date = "date"
+    reactions = "reactions"
+
+
 class MessageFilter(BaseModel):
     before: datetime | None = None
     after: datetime | None = None
     contains: str | None = None
     is_number: bool | None = None
+    sender_name: str | None = None
+    having_reactions: bool | None = None
+
+    class Config:
+        frozen = True

million/model/filter/ParticipantFilter.py

@@ -0,0 +1,23 @@
+from datetime import datetime
+from pydantic import BaseModel
+from enum import Enum
+
+from million.model.filter.MessageFilter import MessageFilter
+
+
+class OrderBy(str, Enum):
+    count = "count"
+
+
+class OrderDirection(str, Enum):
+    asc = "asc"
+    desc = "desc"
+
+
+class ParticipantFilter(MessageFilter):
+
+    order_by: OrderBy | None = OrderBy.count
+    order_dir: OrderDirection | None = OrderDirection.desc
+
+    def __hash__(self):
+        return hash(self.name)

million/model/message.py

@@ -1,7 +1,9 @@
 from datetime import datetime
-from math import floor
 from typing import Any, List
 from pydantic import BaseModel
+import pytz
+
+utc=pytz.UTC
 
 class Reaction(BaseModel):
     reaction: str
@@ -47,7 +49,7 @@ class Message(BaseModel):
 
     @property
     def date(self) -> datetime:
-        return datetime.fromtimestamp(self.timestamp_ms / 1000)
+        return utc.localize(datetime.fromtimestamp(self.timestamp_ms / 1000))
 
     def __str__(self) -> str:
         dt = datetime.fromtimestamp(self.timestamp_ms / 1000)

scripts/find_holes.py

@@ -18,11 +18,12 @@ print(f"Actual counted: {actual_counted}")
 
 holes = find_holes(filtered)
 
-print(len(holes))
 
 for hole in holes:
     print(f"{hole.start() + 1} -> {hole.end() - 1} ({hole.length() - 2})")
 
+print(f"Total holes: {len(holes)}")
+print(f"Total holes size: {sum([h.length() for h in holes if h.length() < 10_000])}")
 
 # lets export a csv file of the holes and the people responsible for them
 with open('output/holes.csv', 'w') as f: