Add script to find holes in counted values

million/analyze/find_holes.py

@@ -0,0 +1,73 @@
+
+
+from typing import List
+from million.model.hole import Hole
+from million.model.message import Message
+from million.model.sequence import Sequence
+
+
+def compute_sequences(messages: List[Message], accepted_max: int = 1_000_000) -> List[Sequence]:
+    sequences: List[Sequence] = []
+    current_sequence = Sequence(
+        start=messages[0].get_counted_value(),
+        start_message=messages[0],
+        end=messages[0].get_counted_value(),
+        end_message=messages[0]
+    )
+    for i in range(1, len(messages)):
+        message = messages[i]
+        message_value = message.get_counted_value()
+        if message_value > accepted_max:
+            continue
+        if message_value - current_sequence.end == 1:
+            current_sequence.end = message_value
+            current_sequence.end_message = message
+        else:
+            sequences.append(current_sequence)
+            current_sequence = Sequence(
+                start=message_value,
+                start_message=message,
+                end=message_value,
+                end_message=message
+            )
+
+    # order the sequences by start
+    sequences.sort(key=lambda s: s.start)
+
+    merged_sequences: List[Sequence] = []
+    current_sequence = sequences[0]
+    for i in range(1, len(sequences)):
+        sequence = sequences[i]
+        sequence_start_is_in_current_sequence = current_sequence.start <= sequence.start and current_sequence.end >= sequence.start
+        sequence_end_is_further = sequence.end > current_sequence.end
+        sequence_start_is_current_end_or_next = sequence.start == current_sequence.end + 1
+
+        if sequence_start_is_in_current_sequence or sequence_start_is_current_end_or_next:
+            if sequence_end_is_further:
+                current_sequence.end = sequence.end
+                current_sequence.end_message = sequence.end_message
+        else:
+            merged_sequences.append(current_sequence)
+            current_sequence = sequence
+
+    # Having merged the sequences once, any sequence having start = end can be removed
+    return [s for s in merged_sequences if s.start != s.end]
+
+
+def find_holes(messages: List[Message], accepted_max: int = 1_000_000) -> List[Hole]:
+    """
+    Find the holes in the conversation
+    """
+    merged_sequences = compute_sequences(messages, accepted_max)
+    holes = []
+    for i in range(1, len(merged_sequences)):
+        previous_sequence = merged_sequences[i - 1]
+        sequence = merged_sequences[i]
+        if sequence.start - previous_sequence.end > 1:
+            holes.append(Hole(
+                start=previous_sequence.end,
+                end=sequence.start,
+                start_message=previous_sequence.end_message,
+                end_message=sequence.start_message
+            ))
+    return holes

million/model/hole.py

@@ -0,0 +1,11 @@
+
+from pydantic import BaseModel
+
+from million.model.message import Message
+
+
+class Hole(BaseModel):
+    start: int
+    end: int
+    start_message: Message
+    end_message: Message

million/model/message.py

@@ -1,4 +1,5 @@
 
+from math import floor
 from typing import Optional
 from pydantic import BaseModel
 
@@ -7,4 +8,19 @@ class Message(BaseModel):
     sender_name: str
     timestamp_ms: int
     content: Optional[str] = None
-    is_geoblocked_for_viewer: Optional[bool] = None
+    is_geoblocked_for_viewer: Optional[bool] = None
+
+    def get_counted_value(self):
+        """
+        The content of the message should be (or contain) a number
+        """
+        value = None
+        # Remove any number that is not a digit
+        # TODO parse potential math expressions in content
+        cleaned_content = ''.join([c for c in self.content if c.isdigit()])
+        try:
+            value = floor(float(cleaned_content))
+        except Exception as e:
+            raise ValueError(
+                f"Message {cleaned_content} does not contain a number ({e})")
+        return value

million/model/sequence.py

@@ -0,0 +1,12 @@
+
+from typing import Optional
+from pydantic import BaseModel
+
+from million.model.message import Message
+
+
+class Sequence(BaseModel):
+    start: int
+    start_message: Message
+    end: int
+    end_message: Message

scripts/find_holes.py

@@ -0,0 +1,46 @@
+from datetime import datetime
+from million.analyze.find_holes import compute_sequences, find_holes
+from million.view.bar_chart import plot as bar_chart
+from million.analyze.count_participations import count_participations
+from million.analyze.retain_counts import retain_counts
+from million.parse.fb_exports import FacebookExportParser
+
+
+DATA_PATH = './data/'
+
+parser = FacebookExportParser()
+
+export = parser.parse(DATA_PATH)
+
+filtered = retain_counts(export.messages)
+
+sequences = compute_sequences(filtered)
+
+actual_counted = sum([s.end - s.start for s in sequences])
+
+print(f"Actual counted: {actual_counted}")
+
+holes = find_holes(filtered)
+
+print(len(holes))
+
+for hole in holes:
+    print(f"{hole.start} - {hole.end} ({hole.end - hole.start})")
+
+
+# lets export a csv file of the holes and the people responsible for them
+with open('holes.csv', 'w') as f:
+    f.write('début,fin,taille,responsable1,responsable2,date1,date2\n')
+    for hole in holes:
+        date_start = datetime.utcfromtimestamp(
+            hole.start_message.timestamp_ms / 1000.0).strftime('%Y-%m-%d %H:%M:%S')
+        date_end = datetime.utcfromtimestamp(
+            hole.end_message.timestamp_ms / 1000.0).strftime('%Y-%m-%d %H:%M:%S')
+        f.write(
+            f"{hole.start},"
+            f"{hole.end},"
+            f"{hole.end - hole.start},"
+            f"{hole.start_message.sender_name},"
+            f"{hole.end_message.sender_name},"
+            f"{date_start},{date_end}\n"
+        )