algorithme alternatif pour trouver les trous

version fonctionnelle

.gitignore

@@ -10,6 +10,8 @@ _build
 .cache
 *.so
 
+output/*
+
 # Installer logs
 pip-log.txt
 

million/analyze/find_holes.py

@@ -1,5 +1,5 @@
 
-
+import time
 from typing import List
 from million.model.hole import Hole
 from million.model.message import Message
@@ -72,52 +72,73 @@ def find_holes(messages: List[Message], accepted_max: int = 1_000_000) -> List[H
             ))
     return holes
 
-def find_holesV2(messages: List[Message]) -> List[Hole]:
+def _find_value_around_index(messages: List[Message], value, idx, amplitude) -> int:
+    check_value = lambda x: messages[x].get_counted_value() == value
+    
+    if check_value(idx): return idx
+
+    for offset in range(1, amplitude):
+        o_idx = idx + offset * +1
+        if check_value(o_idx):
+            return o_idx
+        
+        o_idx = idx + offset * -1
+        if check_value(o_idx):
+            return o_idx
+        
+    return -1
+
+def _open_sequence(sequences: List[Sequence], msg: Message):
+    sequence = Sequence(
+        start=msg.get_counted_value(),
+        start_message=msg,
+        end=-1,
+        end_message=msg
+        )
+    
+    sequences.append(sequence)
+
+def _close_sequence(sequences: List[Sequence]):
+    if len(sequences) == 0: return
+
+    sequences[-1].end = sequences[-1].end_message.get_counted_value()
+
+def _opened_sequence(sequences: List[Sequence]):
+    return len(sequences) > 0 and sequences[-1].end == -1
+
+def find_sequences_v2(messages: List[Message]) -> List[Sequence]:
     current = 1
-    msg_idx = 0
-    threshold = 1000
-    limitAhead = 100
-    limitBehind = 20
+    base_idx = 0
+    amplitude = 200
 
-    holes = []
+    sequences = []
 
-    while msg_idx < len(messages):
-        #search value current in messages from msgIdx, with lookahead then lookbehind
+    while base_idx < len(messages):
+        curr_idx = _find_value_around_index(messages, current, base_idx, amplitude)
+        print(f"searching {current} from [{messages[base_idx]}]\t-> {'Not found' if curr_idx == -1 else 'Itself' if curr_idx == base_idx else messages[curr_idx]}")
         
-        for i in range(0, limitAhead):
-            msgCurrent = messages[msg_idx + i]
-            
-            if msgCurrent.get_counted_value() == current: break
+        if curr_idx != -1: #trouvé
+
+            if not _opened_sequence(sequences):
+                _open_sequence(sequences, messages[curr_idx])
+            else:
+                sequences[-1].end_message = messages[curr_idx]
 
-        if msgCurrent.get_counted_value() != current:
-            for i in range(1, limitBehind):
-                msgCurrent = messages[msg_idx - i]
+            base_idx = curr_idx + 1
+            current += 1
+        else: # pas trouvé
             
-                if msgCurrent.get_counted_value() == current: break
-
-        if msgCurrent.get_counted_value() == current:
-            # la valeur current a été trouvé dans la zone de recherche
-            print(f"{msgCurrent.sender_name} : {msgCurrent.content}")
-            # si un trou était ouvert il faut le fermer
-            if len(holes) > 0 and holes[-1].end == 0:
-                holes[-1].end = current-1
-                holes[-1].end_message = msgCurrent
-                print(f"\t{current-1}")
-            msg_idx += 1
-        else:
-            # la valeur current n'a pas été trouvée
-            # on est dans un trou
-            # si aucun trou n'est ouvert, on en crée un
-            if len(holes) == 0 or holes[-1].end > 0:
-                hole = Hole(
-                    start=current,
-                    end=0,
-                    start_message=messages[msg_idx],
-                    end_message=Message(sender_name='',timestamp_ms=0)
-                    )
-                holes.append(hole)
-                print(f"\t HOLE : {hole.start}\n\t\t...")
-
-        current += 1
-    
-    return holes
+            # fermer la sequence si ouverte
+            if _opened_sequence(sequences):
+                _close_sequence(sequences)
+            
+            if messages[base_idx].get_counted_value() < current:
+                base_idx += 1
+            else:
+                current += 1
+
+
+        #time.sleep(.005)
+
+
+    return sequences

scripts/find_holesV2.py

@@ -1,4 +1,5 @@
-from million.analyze.find_holes import find_holesV2
+from datetime import datetime
+from million.analyze.find_holes import find_sequences_v2
 from million.analyze.retain_counts import retain_counts
 from million.parse.fb_exports import FacebookExportParser
 
@@ -9,8 +10,24 @@ parser = FacebookExportParser()
 export = parser.parse(DATA_PATH)
 filtered = retain_counts(export.messages)
 
-holes = find_holesV2(filtered)
+sequences = find_sequences_v2(filtered)
 
-for hole in holes:
-    print(f"{hole.start} - {hole.end} ({hole.end - hole.start})")
-    print(hole.end_message)
+with open('output/holes.csv', 'w') as f:
+    f.write('début,fin,taille,responsable,date2\n')
+    for i in range(1, len(sequences)):
+        hole_start = sequences[i-1].end+1
+        hole_end = sequences[i].start-1
+        hole_end_message = sequences[i].start_message
+        hole_start_message = sequences[i-1].start_message
+
+        date_start = datetime.utcfromtimestamp(
+            hole_start_message.timestamp_ms / 1000.0).strftime('%Y-%m-%d %H:%M:%S')
+        date_end = datetime.utcfromtimestamp(
+            hole_end_message.timestamp_ms / 1000.0).strftime('%Y-%m-%d %H:%M:%S')
+        f.write(
+            f"{hole_start},"
+            f"{hole_end},"
+            f"{hole_end - hole_start + 1},"
+            f"{hole_end_message.sender_name},"
+            f"{date_start},{date_end}\n"
+        )