+ test script for count analysis

DNS

@@ -1 +0,0 @@
-https://www.youtube.com/watch?v=mC9yute2k_Q 3000

data/DefaultMediaCountMapFile

@@ -0,0 +1,45 @@
+94965738_575896433034148_2204307637284110336_n_2600568116938585.jpg 300
+84316189_161561335269481_4671060857508069376_n_821996424998042.jpg 307
+102407032_3165952896761223_6148002473225081360_n_2378777012422623.jpg 308
+90507136_213915209958557_5413143962586185728_n_1841388439336535.jpg 309
+104035858_782158855522425_8192024435259743235_n_782158852189092.jpg 666
+104123574_322212138769518_5707692183879973515_n_322212135436185.jpg 667
+104434027_271819697398827_5391503491326101448_n_271819694065494.jpg 1312
+95525936_1164142403920772_8318302524985573376_n_299590661169172.jpg 51
+52421236_278953813000944_2106293885134176256_n_2905362479561945.gif 1664
+20688788_286417625173756_1069917705378725888_n_1207474276261488.gif 666
+38234456_1103482206482169_3464478929153163264_n_2628411810759247.gif 1789
+104662968_217216135921364_332403069450983046_n_217216132588031.jpg 1914
+50706165_602050206907030_9269729130708992_n_3180220838709531.gif 1939
+92978153_661083471350851_7802534939089436672_n_585129509051794.jpg 1984
+104872794_272192197333677_7875491468143786127_n_272192194000344.jpg 1995
+49627753_540217636460822_4914566064169287680_n_2394672374158902.png 2048
+https://www.youtube.com/watch?v=mC9yute2k_Q 3000
+65681880_655872084893722_5358758350790066176_n_4051854338219896.gif 3666
+50165487_1987596788210079_254230440078999552_n_1007765122959718.gif 66
+87358105_203182734373455_1323650921388834816_n_3112616325440519.gif 4810
+83527853_509829533251553_144101650338938880_n_1220206091644612.gif 6369
+84441501_209962830394148_963121690001276928_n_736879910407241.gif 6769
+74608941_770967279996317_3169876449326792704_n_1581830875345515.gif 7269
+20505423_878434838980511_4604695143109361664_n_299194367865591.gif 666
+110264758_573811796634371_8422456995004556652_n_782140962427058.gif 666
+65182313_697973310662653_2741056482018590720_n_110058701154052.gif 666
+120437981_961865127657562_2352191202134666388_n_297241988832056.gif 17000
+133574591_2903570269883547_4546172544540158465_n_1987002451464904.gif 18000
+124066484_677676816444243_7811409333876486154_n_382212936779839.gif 20000
+130166493_156814772857168_4400190561706308563_n_1434915410205159.gif 21000
+131881117_200208011820369_5496884526316665472_n_614869492871247.gif 24000
+122477452_404917850669181_7425532495902993743_n_622781605380862.gif 25000
+60398112_324025954936328_3959780282919288832_n_1298063493943852.gif 26400
+132605238_2250185215114171_4387582615384925988_n_1562306254123019.gif 29000
+83715267_525261428388982_9213116445225910272_n_408695157299629.gif 30000
+223698887_1438459123189233_486429511094530589_n_2894947440818658.gif 40000
+245828092_2845723112338892_4090190909716007091_n_2845723109005559.jpg 36399
+246367043_846550806036575_350641140426701499_n_846550802703242.jpg 36400
+247417430_407831760973532_6702356361214642186_n_407831757640199.jpg 36401
+274103826_545381123173322_5027057711080616063_n_545381116506656.jpg 60909
+273830541_343104791045273_1854911206287093351_n_343104784378607.jpg 60910
+273907776_234819785533495_1080142729732940044_n_234819782200162.jpg 60911
+274242881_999574723977290_4022657018268260987_n_999574720643957.jpg 60912
+274008762_640199633904372_3459422682721277586_n_640199623904373.jpg 60913
+audioclip16419453900003855_1094480794455461.mp4 57612

million/analyze/count_analysis.py

@@ -0,0 +1,132 @@
+from typing import List
+import million.analyze.message_evaluation as msg_val
+from million.model.message import Message
+
+
+def check_extra_or_missing_letter(word: str, reference: str) -> bool:
+    """
+    Cette méthode vérifie si la str word contient une et une seule lettre
+    de trop ou de moins par rapport à la str reference
+    """
+    len_word = len(word)
+    len_ref = len(reference)
+
+    if abs(len_word - len_ref) != 1:
+        return False
+
+    shortest = word if len_word < len_ref else reference
+    longest = word if len_word > len_ref else reference
+
+    for i in range(len(shortest)):
+        if shortest[i] != longest[i]:
+            return shortest[i:] == longest[i + 1 :]
+
+    return True
+
+
+def check_single_letter_differ(word: str, reference: str) -> bool:
+    """
+    Cette méthode vérifie si la str word contient une et une seule
+    lettre différente par rapport à la str reference
+    """
+    return sum(1 for x, y in zip(reference, word) if x != y) == 1
+
+
+def check_letter_swap(word: str, reference: str) -> bool:
+    """
+    Cette méthode vérifie si la str word contient un et un seul
+    échange de lettres consécutives par rapport à la str reference
+    """
+    if len(word) != len(reference):
+        return False
+
+    for i in range(len(word) - 1):
+        if word[i] != reference[i]:
+            return word[i + 1] + word[i] + word[i + 2 :] == reference[i:]
+
+    return False
+
+
+def check_typo(word: str, reference: str) -> bool:
+    """
+    Cette méthode vérifie si la str word contient une typo en se référant à la str reference
+    """
+    if len(reference) == len(word):
+        return check_single_letter_differ(word, reference) or check_letter_swap(
+            word, reference
+        )
+    else:
+        return check_extra_or_missing_letter(word, reference)
+
+
+def _check_message_concatenation(messages: List[Message], index: int, expected: int) -> bool:
+    """
+    Cette méthode détermine si la liste messages contient le compte expected à partir de l'index donné
+    en concaténant les valeurs des messages suivants.
+    Cette méthode permet de trouver un compte qui a été étalé sur plusieurs messages
+    """
+    reference = str(expected)
+    testing = ""
+
+    offset = 0
+
+    while len(testing) < len(reference):
+        next_message = messages[index + offset]
+        offset += 1    
+        if next_message.sender_name == messages[index].sender_name:
+            testing += str(msg_val.get(next_message))
+
+    return testing == reference
+
+
+def _heavy_check(messages: List[Message], index: int, expected: int) -> bool:
+    """
+    Cette méthode détermine si la liste messages contient le compte expected à partir de l'index donné.
+    Elle utilise pour cela des méthodes complexes qui ne permettent de trouver un résultat
+    seulement si on est sortis du cas nominal
+    """
+    # TODO
+    #   - créer une méthode pour gérer le cas où plusieurs comptages sont contenus dans le même corps de message
+    #   - créer une méthode pour le cas où les chiffres sont représentés par un substitut au sein du corps du message
+    #     i.e. un nombre écrit en toutes lettres (français ou breton), 🍁 pour 420, @Elias Cheddar pour 69
+    m = messages[index]
+    word = str(msg_val.get(m))
+
+    return _check_message_concatenation(messages, index, expected) or \
+        check_typo(word, str(expected)) and msg_val.get(messages[index+1]) == expected+1
+
+
+def _check_value_around(messages, index, expected, amplitude_after, amplitude_before):
+    for i in range(1, amplitude_after + 1):
+        if index + i < len(messages) and expected == msg_val.get(messages[index + i]):
+            return index + i
+    for i in range(1, amplitude_before + 1):
+        if expected == msg_val.get(messages[index - i]):
+            return index - i
+
+    return None
+
+
+def search_value_at(messages, index, expected, do_heavy_check=True, amplitude_after=1000, amplitude_before=10):
+    """
+    Cette méthode détermine si la liste messages contient le compte expected à partir de l'index donné.
+    Le paramètre amplitude détermine la plage où effectuer les recherches autour de l'index donné.
+    Le paramètre do_heavy_check précise si on doit pousser l'analyse avec des méthodes plus lourdes en cas d'échec
+    """
+    # Si le message courant contient la valeur, on renvoie
+    curr_value = msg_val.get(messages[index])
+    if expected == curr_value:
+        return index
+
+    # Sinon on regarde aux alentours
+    jump_index = _check_value_around(messages, index, expected, amplitude_after, amplitude_before)
+    if jump_index is not None:
+        return jump_index
+
+    # Enfin, si on ne trouve pas la valeur à l'index donné et dans l'amplitude donnée
+    # On performe une vérification lourde à cet endroit
+    if do_heavy_check and _heavy_check(messages, index, expected):
+        return index
+
+    # Si tout cela n'a rien donné, on renvoie None
+    return None

million/analyze/dns_solver.py → million/analyze/media_count_mapper.py

@@ -1,11 +1,12 @@
+from os.path import basename
 from typing import Dict
 
 from pydantic import BaseModel, PrivateAttr
 from million.model.message import Message
 
-_default_file_path = './DNS'
+_default_file_path = 'data/DefaultMediaCountMapFile'
 
-class DNS_solver(BaseModel):
+class MediaCountMapper(BaseModel):
     file_path:str = _default_file_path
 
     _bank: Dict[str, int] | None = PrivateAttr(None)
@@ -33,4 +34,5 @@ class DNS_solver(BaseModel):
         # look into msg attributes
         # find uri
         return (msg.share or None) and msg.share.link or \
-            (msg.gifs or None) and msg.gifs[0].uri
+            (msg.gifs or None) and basename(msg.gifs[0].uri) or \
+            (msg.photos or None) and basename(msg.photos[0].uri)

million/analyze/message_evaluation.py

@@ -1,11 +1,12 @@
 import re
 from typing import Dict
 from million.model.message import Message
-import million.analyze.dns_solver as dns
+import million.analyze.media_count_mapper as mcm
 
 
 _memoization: Dict[Message, int] = {}
-_dns_solver: dns.DNS_solver = dns.DNS_solver()
+_dns_solver: mcm.MediaCountMapper = mcm.MediaCountMapper()
+
 
 def get(msg: Message) -> int:
     """
@@ -13,6 +14,7 @@ def get(msg: Message) -> int:
     """
     return _memoization.get(msg, _compute(msg))
 
+
 def reset(msg: Message) -> None:
     """
     Drop memorized value of this Message
@@ -20,6 +22,7 @@ def reset(msg: Message) -> None:
     if msg in _memoization:
         _memoization.pop(msg)
 
+
 def reset() -> None:
     """
     Drop every memorized message value
@@ -28,20 +31,20 @@ def reset() -> None:
 
 
 def _compute(msg: Message) -> int:
-    value = _dns_solver.solve(msg) or \
-        _computeContent(msg) or \
-        None
+    value = _dns_solver.solve(msg) or _computeContent(msg) or None
 
     _memoization[msg] = value
     return value
 
+
 def _computeContent(msg: Message) -> int:
-    # TODO parse potential math expressions in content
-    match = msg.content and re.search(r"\d+", msg.content)
+    if not msg.content:
+        return
     
+    s = re.sub(r'[^\s\d.,]|[.,]{2,}',"", msg.content)
+    match = re.search(r"\d+", s)
+
     if match:
-        value = int(match.group())
-    else:
-        value = None
-    
-    return value
+        return int(match.group())
+
+    return None

million/model/message.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from typing import Any, List
 from uuid import uuid4
-from pydantic import BaseModel, Field, PrivateAttr, computed_field, validator
+from pydantic import BaseModel, Field, PrivateAttr, computed_field, field_validator
 
 
 class Reaction(BaseModel):
@@ -57,7 +57,19 @@ class Message(BaseModel):
 
     def __str__(self) -> str:
         dt_str = self.date_time.strftime("%d/%m/%Y, %H:%M:%S")
-        return f"{self.sender_name}({dt_str}) : {self.content}"
+
+        msg_str = f"{self.sender_name}({dt_str})"
+
+        if self.content:
+            msg_str += " : " + self.content
+        if self.photos:
+            msg_str += f" [PHOTOS {len(self.photos)}]"
+        if self.videos:
+            msg_str += f" [VIDEOS {len(self.videos)}]"
+        if self.gifs:
+            msg_str += f" [GIFS {len(self.gifs)}]"
+
+        return msg_str
 
     def __hash__(self) -> int:
         return hash(self.item_id)
@@ -67,7 +79,7 @@ class Message(BaseModel):
     def item_id(self) -> str:
         return self._id
 
-    @validator("date_time", pre=True, always=True)
+    @field_validator("date_time")
     def parse_timestamp(cls, v):
         if isinstance(v, int):
             return datetime.fromtimestamp(v / 1000)

scripts/find_missing.py

@@ -0,0 +1,25 @@
+import million.analyze.message_evaluation as msg_val
+import million.parse.fb_exports as fb
+import time
+
+export = fb.parse_dirfiles("./data")
+messages = export.messages
+
+counts = {val for m in messages if (val := msg_val.get(m)) and val <= 1_000_000}
+counts = sorted(counts)
+
+expected_value = 1
+intervals = []
+
+for value in counts:
+    if value != expected_value:
+        interval_length = value - expected_value
+
+        if interval_length == 1:
+            intervals.append(str(expected_value))
+        else:
+            intervals.append(f"{expected_value}..{value - 1}")
+
+    expected_value = value + 1
+
+print(intervals)

scripts/test_count_analysis.py

@@ -0,0 +1,34 @@
+import million.parse.fb_exports as fb
+import million.analyze.message_evaluation as msg_val
+from million.analyze.count_analysis import  search_value_at
+
+
+DATA_PATH = "./data/"
+export = fb.parse_dirfiles(DATA_PATH)
+messages = export.messages
+
+expected = 0
+idx = 0
+total_len = len(messages)
+total_as_percent = 100 / total_len
+
+with open('output/analysis_breakdown.txt', 'w', encoding="utf-8") as fichier:
+    while idx < total_len:
+        print(f"\r{round(idx * total_as_percent, 1)}%", end="")
+
+        # skip messages with no detected value
+        if msg_val.get(messages[idx]) == None:
+            idx += 1
+            continue
+
+        expected += 1
+
+        found_index = search_value_at(messages, idx, expected)
+        
+        if found_index:
+            fichier.write(f"{expected}\t⇒{messages[found_index]}\n")
+            idx = found_index + 1
+        else:
+            fichier.write(f"{expected}[X]\t⇒{messages[idx]}\n")
+
+print("\nComplete analysis in: output/analysis_breakdown.txt")

test/model/message_test.py

@@ -1,28 +1,60 @@
-
-
-from million.model.message import Message
 import million.analyze.message_evaluation as msg_val
 from test.TestCase import TestCase
 
 
 class MessageTest(TestCase):
 
-    def test_message_nominal(self, overrides=None, exclude=None):
+    def test_single_digit(self, overrides=None, exclude=None):
         message = self._message_with_text("1")
 
         assert 1 == msg_val.get(message)
 
+    def test_nothing(self, overrides=None, exclude=None):
+        message = self._message_with_text("")
+
+        assert None == msg_val.get(message)
+
+    def test_message_nominal(self, overrides=None, exclude=None):
+        message = self._message_with_text("1234")
+
+        assert 1234 == msg_val.get(message)
+
     def test_message_with_text(self, overrides=None, exclude=None):
-        message = self._message_with_text("1 text")
+        message = self._message_with_text("... 😏😏 269")
 
-        assert 1 == msg_val.get(message)
+        assert 269 == msg_val.get(message)
 
-    def test_message_floored_dot(self, overrides=None, exclude=None):
-        message = self._message_with_text("1.5")
+    def test_message_with_text_2(self, overrides=None, exclude=None):
+        message = self._message_with_text("331 allez la")
 
-        assert 1 == msg_val.get(message)
+        assert 331 == msg_val.get(message)
 
-    def test_message_floored_comma(self, overrides=None, exclude=None):
-        message = self._message_with_text("1,5")
+    def test_message_with_text_3(self, overrides=None, exclude=None):
+        message = self._message_with_text("Ok 2160")
 
-        assert 1 == msg_val.get(message)
+        assert 2160 == msg_val.get(message)
+
+    def test_message_value_cut(self, overrides=None, exclude=None):
+        message = self._message_with_text("66...😏😏😏9")
+
+        assert 669 == msg_val.get(message)
+
+    def test_message_value_cut_2(self, overrides=None, exclude=None):
+        message = self._message_with_text("82heyyyyyy69")
+
+        assert 8269 == msg_val.get(message)
+
+    def test_message_value_cut_2(self, overrides=None, exclude=None):
+        message = self._message_with_text("9339 9339 9339 9339")
+
+        assert 9339 == msg_val.get(message)
+
+    def test_message_in_middle(self, overrides=None, exclude=None):
+        message = self._message_with_text("A peine 5565 ouais...")
+
+        assert 5565 == msg_val.get(message)
+
+    def test_message_float_1(self, overrides=None, exclude=None):
+        message = self._message_with_text("11111,1111111111111111¼")
+
+        assert 11111 == msg_val.get(message)

test/model/typo_test.py

@@ -0,0 +1,76 @@
+import million.analyze.count_analysis as ca
+from test.TestCase import TestCase
+
+
+class TypoTest(TestCase):
+
+    def test_missing_letter_1(self, overrides=None, exclude=None):
+        assert ca.check_extra_or_missing_letter("4976", "45976") == True
+
+    def test_missing_letter_2(self, overrides=None, exclude=None):
+        assert ca.check_extra_or_missing_letter("4596", "45976") == True
+
+    def test_missing_letter_3(self, overrides=None, exclude=None):
+        assert ca.check_extra_or_missing_letter("5976", "45976") == True
+
+    def test_missing_letter_4(self, overrides=None, exclude=None):
+        assert ca.check_extra_or_missing_letter("4597", "45976") == True
+
+
+    
+    def test_extra_letter_1(self, overrides=None, exclude=None):
+        assert ca.check_extra_or_missing_letter("459766", "45976") == True
+
+    def test_extra_letter_2(self, overrides=None, exclude=None):
+        assert ca.check_extra_or_missing_letter("545976", "45976") == True
+
+    def test_extra_letter_3(self, overrides=None, exclude=None):
+        assert ca.check_extra_or_missing_letter("452976", "45976") == True
+
+    def test_extra_letter_4(self, overrides=None, exclude=None):
+        assert ca.check_extra_or_missing_letter("459776", "45976") == True
+
+    def test_extra_letter_5(self, overrides=None, exclude=None):
+        assert ca.check_extra_or_missing_letter("45976", "45976") == False
+
+
+    def test_single_letter_differ_1(self, overrides=None, exclude=None):
+        assert ca.check_single_letter_differ("35976", "45976") == True
+
+    def test_single_letter_differ_2(self, overrides=None, exclude=None):
+        assert ca.check_single_letter_differ("45986", "45976") == True
+
+    def test_single_letter_differ_3(self, overrides=None, exclude=None):
+        assert ca.check_single_letter_differ("44986", "45976") == False
+
+    def test_single_letter_differ_4(self, overrides=None, exclude=None):
+        assert ca.check_single_letter_differ("35975", "45976") == False
+
+    def test_single_letter_differ_5(self, overrides=None, exclude=None):
+        assert ca.check_single_letter_differ("4976", "45976") == False
+
+    def test_single_letter_differ_6(self, overrides=None, exclude=None):
+        assert ca.check_single_letter_differ("4597", "45976") == False
+        
+    def test_single_letter_differ_7(self, overrides=None, exclude=None):
+        assert ca.check_single_letter_differ("45976", "45976") == False
+
+
+    
+    def test_letter_swap_1(self, overrides=None, exclude=None):
+        assert ca.check_letter_swap("45976", "45976") == False
+
+    def test_letter_swap_2(self, overrides=None, exclude=None):
+        assert ca.check_letter_swap("49576", "45976") == True
+
+    def test_letter_swap_3(self, overrides=None, exclude=None):
+        assert ca.check_letter_swap("45967", "45976") == True
+
+    def test_letter_swap_4(self, overrides=None, exclude=None):
+        assert ca.check_letter_swap("47956", "45976") == False
+
+    def test_letter_swap_5(self, overrides=None, exclude=None):
+        assert ca.check_letter_swap("54966", "45976") == False
+
+    def test_letter_swap_6(self, overrides=None, exclude=None):
+        assert ca.check_letter_swap("54967", "45976") == False