Dockerfile

@@ -4,6 +4,7 @@ FROM python:3.11-buster
 RUN pip install poetry
 
 COPY pyproject.toml poetry.lock ./
+COPY ./data/DefaultMediaCountMapFile ./data/DefaultMediaCountMapFile
 COPY ./million ./million
 COPY ./test ./test
 

data/DefaultMediaCountMapFile

@@ -0,0 +1,45 @@
+94965738_575896433034148_2204307637284110336_n_2600568116938585.jpg 300
+84316189_161561335269481_4671060857508069376_n_821996424998042.jpg 307
+102407032_3165952896761223_6148002473225081360_n_2378777012422623.jpg 308
+90507136_213915209958557_5413143962586185728_n_1841388439336535.jpg 309
+104035858_782158855522425_8192024435259743235_n_782158852189092.jpg 666
+104123574_322212138769518_5707692183879973515_n_322212135436185.jpg 667
+104434027_271819697398827_5391503491326101448_n_271819694065494.jpg 1312
+95525936_1164142403920772_8318302524985573376_n_299590661169172.jpg 51
+52421236_278953813000944_2106293885134176256_n_2905362479561945.gif 1664
+20688788_286417625173756_1069917705378725888_n_1207474276261488.gif 666
+38234456_1103482206482169_3464478929153163264_n_2628411810759247.gif 1789
+104662968_217216135921364_332403069450983046_n_217216132588031.jpg 1914
+50706165_602050206907030_9269729130708992_n_3180220838709531.gif 1939
+92978153_661083471350851_7802534939089436672_n_585129509051794.jpg 1984
+104872794_272192197333677_7875491468143786127_n_272192194000344.jpg 1995
+49627753_540217636460822_4914566064169287680_n_2394672374158902.png 2048
+https://www.youtube.com/watch?v=mC9yute2k_Q 3000
+65681880_655872084893722_5358758350790066176_n_4051854338219896.gif 3666
+50165487_1987596788210079_254230440078999552_n_1007765122959718.gif 66
+87358105_203182734373455_1323650921388834816_n_3112616325440519.gif 4810
+83527853_509829533251553_144101650338938880_n_1220206091644612.gif 6369
+84441501_209962830394148_963121690001276928_n_736879910407241.gif 6769
+74608941_770967279996317_3169876449326792704_n_1581830875345515.gif 7269
+20505423_878434838980511_4604695143109361664_n_299194367865591.gif 666
+110264758_573811796634371_8422456995004556652_n_782140962427058.gif 666
+65182313_697973310662653_2741056482018590720_n_110058701154052.gif 666
+120437981_961865127657562_2352191202134666388_n_297241988832056.gif 17000
+133574591_2903570269883547_4546172544540158465_n_1987002451464904.gif 18000
+124066484_677676816444243_7811409333876486154_n_382212936779839.gif 20000
+130166493_156814772857168_4400190561706308563_n_1434915410205159.gif 21000
+131881117_200208011820369_5496884526316665472_n_614869492871247.gif 24000
+122477452_404917850669181_7425532495902993743_n_622781605380862.gif 25000
+60398112_324025954936328_3959780282919288832_n_1298063493943852.gif 26400
+132605238_2250185215114171_4387582615384925988_n_1562306254123019.gif 29000
+83715267_525261428388982_9213116445225910272_n_408695157299629.gif 30000
+223698887_1438459123189233_486429511094530589_n_2894947440818658.gif 40000
+245828092_2845723112338892_4090190909716007091_n_2845723109005559.jpg 36399
+246367043_846550806036575_350641140426701499_n_846550802703242.jpg 36400
+247417430_407831760973532_6702356361214642186_n_407831757640199.jpg 36401
+274103826_545381123173322_5027057711080616063_n_545381116506656.jpg 60909
+273830541_343104791045273_1854911206287093351_n_343104784378607.jpg 60910
+273907776_234819785533495_1080142729732940044_n_234819782200162.jpg 60911
+274242881_999574723977290_4022657018268260987_n_999574720643957.jpg 60912
+274008762_640199633904372_3459422682721277586_n_640199623904373.jpg 60913
+audioclip16419453900003855_1094480794455461.mp4 57612

million/analyze/count_analysis.py

@@ -0,0 +1,132 @@
+from typing import List
+import million.analyze.message_evaluation as msg_val
+from million.model.message import Message
+
+
+def check_extra_or_missing_letter(word: str, reference: str) -> bool:
+    """
+    Cette méthode vérifie si la str word contient une et une seule lettre
+    de trop ou de moins par rapport à la str reference
+    """
+    len_word = len(word)
+    len_ref = len(reference)
+
+    if abs(len_word - len_ref) != 1:
+        return False
+
+    shortest = word if len_word < len_ref else reference
+    longest = word if len_word > len_ref else reference
+
+    for i in range(len(shortest)):
+        if shortest[i] != longest[i]:
+            return shortest[i:] == longest[i + 1 :]
+
+    return True
+
+
+def check_single_letter_differ(word: str, reference: str) -> bool:
+    """
+    Cette méthode vérifie si la str word contient une et une seule
+    lettre différente par rapport à la str reference
+    """
+    return sum(1 for x, y in zip(reference, word) if x != y) == 1
+
+
+def check_letter_swap(word: str, reference: str) -> bool:
+    """
+    Cette méthode vérifie si la str word contient un et un seul
+    échange de lettres consécutives par rapport à la str reference
+    """
+    if len(word) != len(reference):
+        return False
+
+    for i in range(len(word) - 1):
+        if word[i] != reference[i]:
+            return word[i + 1] + word[i] + word[i + 2 :] == reference[i:]
+
+    return False
+
+
+def check_typo(word: str, reference: str) -> bool:
+    """
+    Cette méthode vérifie si la str word contient une typo en se référant à la str reference
+    """
+    if len(reference) == len(word):
+        return check_single_letter_differ(word, reference) or check_letter_swap(
+            word, reference
+        )
+    else:
+        return check_extra_or_missing_letter(word, reference)
+
+
+def _check_message_concatenation(messages: List[Message], index: int, expected: int) -> bool:
+    """
+    Cette méthode détermine si la liste messages contient le compte expected à partir de l'index donné
+    en concaténant les valeurs des messages suivants.
+    Cette méthode permet de trouver un compte qui a été étalé sur plusieurs messages
+    """
+    reference = str(expected)
+    testing = ""
+
+    offset = 0
+
+    while len(testing) < len(reference):
+        next_message = messages[index + offset]
+        offset += 1    
+        if next_message.sender_name == messages[index].sender_name:
+            testing += str(msg_val.get(next_message))
+
+    return testing == reference
+
+
+def _heavy_check(messages: List[Message], index: int, expected: int) -> bool:
+    """
+    Cette méthode détermine si la liste messages contient le compte expected à partir de l'index donné.
+    Elle utilise pour cela des méthodes complexes qui ne permettent de trouver un résultat
+    seulement si on est sortis du cas nominal
+    """
+    # TODO
+    #   - créer une méthode pour gérer le cas où plusieurs comptages sont contenus dans le même corps de message
+    #   - créer une méthode pour le cas où les chiffres sont représentés par un substitut au sein du corps du message
+    #     i.e. un nombre écrit en toutes lettres (français ou breton), 🍁 pour 420, @Elias Cheddar pour 69
+    m = messages[index]
+    word = str(msg_val.get(m))
+
+    return _check_message_concatenation(messages, index, expected) or \
+        check_typo(word, str(expected)) and msg_val.get(messages[index+1]) == expected+1
+
+
+def _check_value_around(messages, index, expected, amplitude_after, amplitude_before):
+    for i in range(1, amplitude_after + 1):
+        if index + i < len(messages) and expected == msg_val.get(messages[index + i]):
+            return index + i
+    for i in range(1, amplitude_before + 1):
+        if expected == msg_val.get(messages[index - i]):
+            return index - i
+
+    return None
+
+
+def search_value_at(messages, index, expected, do_heavy_check=True, amplitude_after=1000, amplitude_before=10):
+    """
+    Cette méthode détermine si la liste messages contient le compte expected à partir de l'index donné.
+    Le paramètre amplitude détermine la plage où effectuer les recherches autour de l'index donné.
+    Le paramètre do_heavy_check précise si on doit pousser l'analyse avec des méthodes plus lourdes en cas d'échec
+    """
+    # Si le message courant contient la valeur, on renvoie
+    curr_value = msg_val.get(messages[index])
+    if expected == curr_value:
+        return index
+
+    # Sinon on regarde aux alentours
+    jump_index = _check_value_around(messages, index, expected, amplitude_after, amplitude_before)
+    if jump_index is not None:
+        return jump_index
+
+    # Enfin, si on ne trouve pas la valeur à l'index donné et dans l'amplitude donnée
+    # On performe une vérification lourde à cet endroit
+    if do_heavy_check and _heavy_check(messages, index, expected):
+        return index
+
+    # Si tout cela n'a rien donné, on renvoie None
+    return None

million/analyze/count_participations.py

@@ -1,23 +1,30 @@
-
-from typing import List
+from collections import Counter
+from typing import Dict, List
 from million.model.message import Message
 from million.model.participant import Participant
 
 
-def count_participations(messages: List[Message], participants: List[Participant]):
+def count_participations(
+        messages: List[Message],
+        participants: List[Participant] | None = [],
+        threshold: int | None = 0
+        ) -> Dict[str, int]:
     """
-    Count the number of messages sent by each participant
+    Count the number of messages sent by each participant,\n
+    you can specify a threshold to return only people having reached that many counts
     """
-    participations = {}
-    for participant in participants:
-        participations[participant.name] = 0
-
-    for message in messages:
-        if message.sender_name not in participations:
-            participations[message.sender_name] = 1
-        else:
-            participations[message.sender_name] += 1
+    participations = dict.fromkeys([p.name for p in participants], 0)
+    participations.update(Counter([m.sender_name for m in messages]))
+    
+    return {k: v for k,v in sorted(participations.items(), key=lambda x: -x[1]) if v >= threshold}
 
-    ordered_participations = sorted(
-        participations.items(), key=lambda x: x[1], reverse=True)
-    return [{"name": v[0], "participations": v[1]} for v in ordered_participations]
+def podium(
+        messages: List[Message],
+        top: int,
+        participants: List[Participant] | None = [],
+        ) -> Dict[str, int]:
+    """
+    Returns the N biggest counters
+    """
+    cp = count_participations(messages, participants)
+    return {k: cp[k] for idx, k in enumerate(cp) if idx < top}

million/analyze/find_holes.py

@@ -1,5 +1,3 @@
-
-
 from typing import List
 from million.model.message import Message
 from million.model.sequence import Sequence
@@ -23,7 +21,6 @@ def compute_sequences(messages: List[Message], accepted_max: int = 1_000_000) ->
 
     return sequences            
 
-
 def merge_duplicates(sequences: List[Sequence]) -> List[Sequence]:
     """ 
     Take sequences as an input and returns a list with every
@@ -43,8 +40,6 @@ def merge_duplicates(sequences: List[Sequence]) -> List[Sequence]:
 
     return result
 
-
-
 def invert_sequences(sequences: List[Sequence]) -> List[Sequence]:
     """ 
     Returns the sequences representing the spaces between

million/analyze/media_count_mapper.py

@@ -0,0 +1,38 @@
+from os.path import basename
+from typing import Dict
+
+from pydantic import BaseModel, PrivateAttr
+from million.model.message import Message
+
+_default_file_path = 'data/DefaultMediaCountMapFile'
+
+class MediaCountMapper(BaseModel):
+    file_path:str = _default_file_path
+
+    _bank: Dict[str, int] | None = PrivateAttr(None)
+
+    def solve(self, msg: Message) -> int:
+        if self._bank == None: 
+            self._bank = self.load(self.file_path)
+
+        k = self._get_key(msg)
+        if k and k in self._bank: return self._bank[k]
+
+        return None
+
+    def load(self, file_name: str) -> Dict[str, int]:
+        result = {}
+        with open(file_name, 'r') as f:
+            for line in f:
+                a,b = line.split()
+                result[a] = int(b)
+
+        return result
+
+    def _get_key(self, msg: Message) -> str:
+
+        # look into msg attributes
+        # find uri
+        return (msg.share or None) and msg.share.link or \
+            (msg.gifs or None) and basename(msg.gifs[0].uri) or \
+            (msg.photos or None) and basename(msg.photos[0].uri)

million/analyze/message_evaluation.py

@@ -1,38 +1,50 @@
-from math import floor
 import re
 from typing import Dict
 from million.model.message import Message
+import million.analyze.media_count_mapper as mcm
 
-memoization: Dict[Message, int] = {}
 
-# TODO WIP
-# - DNS to resolve audio, gif, pictures with counts
-def __compute__(msg: Message) -> int:
-    value = __computeContent(msg)
+_memoization: Dict[Message, int] = {}
+_dns_solver: mcm.MediaCountMapper = mcm.MediaCountMapper()
 
-    memoization[msg] = value
-    return value
 
-def __computeContent(msg: Message) -> int:
-    # TODO parse potential math expressions in content
-    match = re.search(r"\d+", msg.content)
-    
-    if match:
-        value = int(match[0])
-    else:
-        value = None
-    
-    return value
+def get(msg: Message) -> int:
+    """
+    Returns the estimated value counted in this message
+    """
+    return _memoization.get(msg, _compute(msg))
+
 
 def reset(msg: Message) -> None:
-    if msg in memoization:
-        memoization.pop(msg)
+    """
+    Drop memorized value of this Message
+    """
+    if msg in _memoization:
+        _memoization.pop(msg)
 
-def reset() -> None:
-    memoization.clear()
 
-def get(msg: Message) -> int:
+def reset() -> None:
     """
-    Returns the estimated value counted in this message
+    Drop every memorized message value
     """
-    return memoization.get(msg, __compute__(msg))
+    _memoization.clear()
+
+
+def _compute(msg: Message) -> int:
+    value = _dns_solver.solve(msg) or _computeContent(msg) or None
+
+    _memoization[msg] = value
+    return value
+
+
+def _computeContent(msg: Message) -> int:
+    if not msg.content:
+        return
+    
+    s = re.sub(r'[^\s\d.,]|[.,]{2,}',"", msg.content)
+    match = re.search(r"\d+", s)
+
+    if match:
+        return int(match.group())
+
+    return None

million/analyze/retain_counts.py

@@ -1,15 +0,0 @@
-
-import re
-from typing import List
-from million.model.message import Message
-
-
-def retain_counts(messages : List[Message])-> List[Message]:
-    """
-    Retain only the messages that have a content
-    """
-    return [
-        m for m in messages 
-        if m.content and
-        re.search('(\d{2,}|^\d$)', m.content)
-        ]

million/analyze/word_finder.py

@@ -1,11 +1,51 @@
+from datetime import date
 import re
+import million.analyze.message_evaluation as msg_val
 from typing import List
 from million.model.message import Message
 
 
-def _wordFilter(msg: Message, words: List[str]) -> bool:
-    rgx = r"(\b"+ r'\b|\b'.join(words) + r"\b)"
-    return msg.content and re.search(rgx, msg.content, re.I)
+def filter_words(messages: List[Message], words: List[str]) -> List[Message]:
+    """
+    Return every message containg the given words, you can use regex syntax inside your words
+    i.e. find_words(messages, ["dogs?","m(ous|ic)e"])
+    will search for : dog, dogs, mouse, mice
+    """
+    r_words = [rf"\b{w}\b" for w in words]
+    rgx = "(" + "|".join(r_words) + ")"
+    return [m for m in messages if m.content and re.search(rgx, m.content, re.I)]
+
+
+def filter_value(messages: List[Message], val: int) -> List[Message]:
+    """
+    Return every message whose value is evaluated to the given val
+    """
+    return [m for m in messages if msg_val.get(m) == val]
+
+
+def filter_date(messages: List[Message], other: date) -> List[Message]:
+    """
+    Return every message posted on the given date
+    """
+    return [m for m in messages if m.date_time.date() == other]
+
+
+def filter_neighbours(
+    messages: List[Message], msg: Message, amplitude: int = 10
+) -> List[Message]:
+    """
+    Return the messages posted just before and after the given one.
+    amplitude will indicate how many messages to look for in each direction
+    """
+    idx = messages.index(msg)
+    start_index = max(0, idx - amplitude)
+    end_index = min(len(messages), idx + amplitude + 1)
+    return messages[start_index:end_index]
+
+
+def retain_counts(messages: List[Message]) -> List[Message]:
+    """
+    Retain only the messages for which are considered having a counted value
+    """
+    return [msg for msg in messages if msg_val.get(msg)]
 
-def findWords(messages: List[Message], words: List[str]) -> List[Message]:
-    return filter(lambda m: _wordFilter(m, words), messages)

million/model/fb_export.py

@@ -1,11 +1,10 @@
 from __future__ import annotations
-
 from typing import Any, List, Set
 from pydantic import BaseModel
-
 from million.model.message import Message
 from million.model.participant import Participant
 
+
 class Image(BaseModel):
     creation_timestamp: int
     uri: str
@@ -28,6 +27,7 @@ class FacebookExport(BaseModel):
     image: Image
     joinable_mode: JoinableMode
 
+
     def merge(self, other: FacebookExport) -> None:
         if self == other:
             self.messages.extend(other.messages)
@@ -35,9 +35,10 @@ class FacebookExport(BaseModel):
             self.magic_words.update(other.magic_words)
 
     def sort(self) -> None:
-        self.messages.sort(key = lambda m: m.timestamp_ms)
+        self.messages.sort(key = lambda m: m.date_time)
+
 
-    # NOTE Toughen equality conditions ?
     def __eq__(self, other: FacebookExport) -> bool:
+        # NOTE Toughen equality conditions ?
         return self.title == other.title \
             and self.image == other.image

million/model/message.py

@@ -1,38 +1,46 @@
 from datetime import datetime
-from math import floor
 from typing import Any, List
-from pydantic import BaseModel
+from uuid import uuid4
+from pydantic import BaseModel, Field, PrivateAttr, computed_field, field_validator
+
 
 class Reaction(BaseModel):
     reaction: str
     actor: str
 
+
 class AudioFile(BaseModel):
     uri: str
     creation_timestamp: int
 
+
 class Video(BaseModel):
     uri: str
     creation_timestamp: int
 
+
 class Photo(BaseModel):
     uri: str
     creation_timestamp: int
 
+
 class Gif(BaseModel):
     uri: str
 
+
 class Share(BaseModel):
     link: str
     share_text: str
 
+
 class Sticker(BaseModel):
     uri: str
     ai_stickers: List[Any]
 
+
 class Message(BaseModel):
     sender_name: str
-    timestamp_ms: int
+    date_time: datetime = Field(alias="timestamp_ms")
     content: str | None = None
     sticker: Sticker | None = None
     share: Share | None = None
@@ -45,10 +53,34 @@ class Message(BaseModel):
     is_unsent: bool | None = None
     is_geoblocked_for_viewer: bool
 
+    _id: str = PrivateAttr(default_factory=lambda: str(uuid4()))
+
     def __str__(self) -> str:
-        dt = datetime.fromtimestamp(self.timestamp_ms / 1000)
-        dt_str = dt.strftime("%d/%m/%Y, %H:%M:%S")
-        return f"{self.sender_name}({dt_str}) : {self.content}"
+        dt_str = self.date_time.strftime("%d/%m/%Y, %H:%M:%S")
+
+        msg_str = f"{self.sender_name}({dt_str})"
+
+        if self.content:
+            msg_str += " : " + self.content
+        if self.photos:
+            msg_str += f" [PHOTOS {len(self.photos)}]"
+        if self.videos:
+            msg_str += f" [VIDEOS {len(self.videos)}]"
+        if self.gifs:
+            msg_str += f" [GIFS {len(self.gifs)}]"
+
+        return msg_str
 
     def __hash__(self) -> int:
-        return hash(self.sender_name + str(self.timestamp_ms))
+        return hash(self.item_id)
+
+    @computed_field
+    @property
+    def item_id(self) -> str:
+        return self._id
+
+    @field_validator("date_time")
+    def parse_timestamp(cls, v):
+        if isinstance(v, int):
+            return datetime.fromtimestamp(v / 1000)
+        return v

million/model/sequence.py

@@ -1,8 +1,5 @@
 from __future__ import annotations
-
-from pydantic import BaseModel
-import pydantic
-
+from pydantic import validator, BaseModel
 from million.model.message import Message
 import million.analyze.message_evaluation as msg_val
 
@@ -11,7 +8,7 @@ class Sequence(BaseModel):
     start_message: Message
     end_message: Message | None = None
 
-    @pydantic.validator('end_message', pre=True, always=True)
+    @validator('end_message', pre=True, always=True)
     def default_end_message(cls, v, *, values):
         return v or values['start_message'] 
 

million/parse/fb_exports.py

@@ -48,6 +48,7 @@ def parse_dirfiles(file_dir: str) -> FacebookExport:
     result.sort()
     return result
 
+
 def __read_broken_fb_json(binary_data):
     # https://stackoverflow.com/questions/50008296/facebook-json-badly-encoded
     repaired = re.sub(

scripts/find_gromots.py

@@ -1,14 +1,8 @@
-from datetime import datetime
-from million.analyze.word_finder import findWords
-from million.parse.fb_exports import FacebookExportParser
+from million.analyze.word_finder import filter_words
+import million.parse.fb_exports as fb
 
 
 DATA_PATH = './data/'
-
-parser = FacebookExportParser()
-
-export = parser.parse(DATA_PATH)
-
 gros_mots = [
     '.*merde.*',
     'sexe',
@@ -30,7 +24,8 @@ gros_mots = [
     'pti?n'
     ]
 
-msg_gros_mots = findWords(export.messages, gros_mots)
+export = fb.parse_dirfiles(DATA_PATH)
+msg_gros_mots = filter_words(export.messages, gros_mots)
 
 msg_gros_mots_grp = {}
 
@@ -42,6 +37,5 @@ for name in sorted(msg_gros_mots_grp, key = lambda k: len(msg_gros_mots_grp[k]))
     print(name)
 
     for msg in msg_gros_mots_grp[name]:
-        time = datetime.fromtimestamp(msg.timestamp_ms / 1000)
-        time_str = time.strftime("%d/%m/%Y %H:%M:%S")
+        time_str = msg.date_time.strftime("%d/%m/%Y %H:%M:%S")
         print(f"\t{time_str} : {msg.content}")

scripts/find_holes.py

@@ -1,22 +1,24 @@
 from datetime import datetime
-from million.analyze.find_holes import compute_sequences, find_holes
-from million.analyze.retain_counts import retain_counts
+import million.analyze.find_holes as fh
+from million.analyze.word_finder import retain_counts
 import million.parse.fb_exports as fb
 
 
-DATA_PATH = './data/'
+DATA_PATH = "./data/"
 
 export = fb.parse_dirfiles(DATA_PATH)
 
 filtered = retain_counts(export.messages)
 
-sequences = compute_sequences(filtered)
+sequences = fh.compute_sequences(filtered)
 
 actual_counted = sum([s.length() for s in sequences])
 
 print(f"Actual counted: {actual_counted}")
 
-holes = find_holes(filtered)
+merged = fh.merge_duplicates(sequences)
+merged = [s for s in merged if s.length() > 1]
+holes = fh.find_holes(filtered)
 
 print(len(holes))
 
@@ -25,13 +27,11 @@ for hole in holes:
 
 
 # lets export a csv file of the holes and the people responsible for them
-with open('output/holes.csv', 'w') as f:
-    f.write('début,fin,taille,responsable1,responsable2,date1,date2\n')
+with open("output/holes.csv", "w") as f:
+    f.write("début,fin,taille,responsable1,responsable2,date1,date2\n")
     for hole in holes:
-        date_start = datetime.utcfromtimestamp(
-            hole.start_message.timestamp_ms / 1000.0).strftime('%Y-%m-%d %H:%M:%S')
-        date_end = datetime.utcfromtimestamp(
-            hole.end_message.timestamp_ms / 1000.0).strftime('%Y-%m-%d %H:%M:%S')
+        date_start = hole.start_message.date_time.strftime("%Y-%m-%d %H:%M:%S")
+        date_end = hole.end_message.date_time.strftime("%Y-%m-%d %H:%M:%S")
         f.write(
             f"{hole.start()},"
             f"{hole.end()},"

scripts/find_missing.py

@@ -0,0 +1,25 @@
+import million.analyze.message_evaluation as msg_val
+import million.parse.fb_exports as fb
+import time
+
+export = fb.parse_dirfiles("./data")
+messages = export.messages
+
+counts = {val for m in messages if (val := msg_val.get(m)) and val <= 1_000_000}
+counts = sorted(counts)
+
+expected_value = 1
+intervals = []
+
+for value in counts:
+    if value != expected_value:
+        interval_length = value - expected_value
+
+        if interval_length == 1:
+            intervals.append(str(expected_value))
+        else:
+            intervals.append(f"{expected_value}..{value - 1}")
+
+    expected_value = value + 1
+
+print(intervals)

scripts/read_top.py

@@ -1,21 +1,16 @@
-from million.view.bar_chart import plot as bar_chart
 from million.analyze.count_participations import count_participations
-from million.analyze.retain_counts import retain_counts
+from million.analyze.word_finder import retain_counts
 import million.parse.fb_exports as fb
 
 
 DATA_PATH = './data/'
 
 export = fb.parse_dirfiles(DATA_PATH)
-
 filtered = retain_counts(export.messages)
 
 print(len(filtered))
 
-counted_participations = count_participations(filtered, export.participants)
-
-kept_participations = [
-    p for p in counted_participations if p['participations'] > 100]
+participations = count_participations(filtered, export.participants, 100)
 
-print("\n".join(
-    [f"{p['name']}: {p['participations']}" for p in kept_participations]))
+for name, count in participations.items():
+    print(f"{name}: {count}")

scripts/test_count_analysis.py

@@ -0,0 +1,34 @@
+import million.parse.fb_exports as fb
+import million.analyze.message_evaluation as msg_val
+from million.analyze.count_analysis import  search_value_at
+
+
+DATA_PATH = "./data/"
+export = fb.parse_dirfiles(DATA_PATH)
+messages = export.messages
+
+expected = 0
+idx = 0
+total_len = len(messages)
+total_as_percent = 100 / total_len
+
+with open('output/analysis_breakdown.txt', 'w', encoding="utf-8") as fichier:
+    while idx < total_len:
+        print(f"\r{round(idx * total_as_percent, 1)}%", end="")
+
+        # skip messages with no detected value
+        if msg_val.get(messages[idx]) == None:
+            idx += 1
+            continue
+
+        expected += 1
+
+        found_index = search_value_at(messages, idx, expected)
+        
+        if found_index:
+            fichier.write(f"{expected}\t⇒{messages[found_index]}\n")
+            idx = found_index + 1
+        else:
+            fichier.write(f"{expected}[X]\t⇒{messages[idx]}\n")
+
+print("\nComplete analysis in: output/analysis_breakdown.txt")

test/model/message_test.py

@@ -1,28 +1,60 @@
-
-
-from million.model.message import Message
 import million.analyze.message_evaluation as msg_val
 from test.TestCase import TestCase
 
 
 class MessageTest(TestCase):
 
-    def test_message_nominal(self, overrides=None, exclude=None):
+    def test_single_digit(self, overrides=None, exclude=None):
         message = self._message_with_text("1")
 
         assert 1 == msg_val.get(message)
 
+    def test_nothing(self, overrides=None, exclude=None):
+        message = self._message_with_text("")
+
+        assert None == msg_val.get(message)
+
+    def test_message_nominal(self, overrides=None, exclude=None):
+        message = self._message_with_text("1234")
+
+        assert 1234 == msg_val.get(message)
+
     def test_message_with_text(self, overrides=None, exclude=None):
-        message = self._message_with_text("1 text")
+        message = self._message_with_text("... 😏😏 269")
 
-        assert 1 == msg_val.get(message)
+        assert 269 == msg_val.get(message)
 
-    def test_message_floored_dot(self, overrides=None, exclude=None):
-        message = self._message_with_text("1.5")
+    def test_message_with_text_2(self, overrides=None, exclude=None):
+        message = self._message_with_text("331 allez la")
 
-        assert 1 == msg_val.get(message)
+        assert 331 == msg_val.get(message)
 
-    def test_message_floored_comma(self, overrides=None, exclude=None):
-        message = self._message_with_text("1,5")
+    def test_message_with_text_3(self, overrides=None, exclude=None):
+        message = self._message_with_text("Ok 2160")
 
-        assert 1 == msg_val.get(message)
+        assert 2160 == msg_val.get(message)
+
+    def test_message_value_cut(self, overrides=None, exclude=None):
+        message = self._message_with_text("66...😏😏😏9")
+
+        assert 669 == msg_val.get(message)
+
+    def test_message_value_cut_2(self, overrides=None, exclude=None):
+        message = self._message_with_text("82heyyyyyy69")
+
+        assert 8269 == msg_val.get(message)
+
+    def test_message_value_cut_2(self, overrides=None, exclude=None):
+        message = self._message_with_text("9339 9339 9339 9339")
+
+        assert 9339 == msg_val.get(message)
+
+    def test_message_in_middle(self, overrides=None, exclude=None):
+        message = self._message_with_text("A peine 5565 ouais...")
+
+        assert 5565 == msg_val.get(message)
+
+    def test_message_float_1(self, overrides=None, exclude=None):
+        message = self._message_with_text("11111,1111111111111111¼")
+
+        assert 11111 == msg_val.get(message)

test/model/typo_test.py

@@ -0,0 +1,76 @@
+import million.analyze.count_analysis as ca
+from test.TestCase import TestCase
+
+
+class TypoTest(TestCase):
+
+    def test_missing_letter_1(self, overrides=None, exclude=None):
+        assert ca.check_extra_or_missing_letter("4976", "45976") == True
+
+    def test_missing_letter_2(self, overrides=None, exclude=None):
+        assert ca.check_extra_or_missing_letter("4596", "45976") == True
+
+    def test_missing_letter_3(self, overrides=None, exclude=None):
+        assert ca.check_extra_or_missing_letter("5976", "45976") == True
+
+    def test_missing_letter_4(self, overrides=None, exclude=None):
+        assert ca.check_extra_or_missing_letter("4597", "45976") == True
+
+
+    
+    def test_extra_letter_1(self, overrides=None, exclude=None):
+        assert ca.check_extra_or_missing_letter("459766", "45976") == True
+
+    def test_extra_letter_2(self, overrides=None, exclude=None):
+        assert ca.check_extra_or_missing_letter("545976", "45976") == True
+
+    def test_extra_letter_3(self, overrides=None, exclude=None):
+        assert ca.check_extra_or_missing_letter("452976", "45976") == True
+
+    def test_extra_letter_4(self, overrides=None, exclude=None):
+        assert ca.check_extra_or_missing_letter("459776", "45976") == True
+
+    def test_extra_letter_5(self, overrides=None, exclude=None):
+        assert ca.check_extra_or_missing_letter("45976", "45976") == False
+
+
+    def test_single_letter_differ_1(self, overrides=None, exclude=None):
+        assert ca.check_single_letter_differ("35976", "45976") == True
+
+    def test_single_letter_differ_2(self, overrides=None, exclude=None):
+        assert ca.check_single_letter_differ("45986", "45976") == True
+
+    def test_single_letter_differ_3(self, overrides=None, exclude=None):
+        assert ca.check_single_letter_differ("44986", "45976") == False
+
+    def test_single_letter_differ_4(self, overrides=None, exclude=None):
+        assert ca.check_single_letter_differ("35975", "45976") == False
+
+    def test_single_letter_differ_5(self, overrides=None, exclude=None):
+        assert ca.check_single_letter_differ("4976", "45976") == False
+
+    def test_single_letter_differ_6(self, overrides=None, exclude=None):
+        assert ca.check_single_letter_differ("4597", "45976") == False
+        
+    def test_single_letter_differ_7(self, overrides=None, exclude=None):
+        assert ca.check_single_letter_differ("45976", "45976") == False
+
+
+    
+    def test_letter_swap_1(self, overrides=None, exclude=None):
+        assert ca.check_letter_swap("45976", "45976") == False
+
+    def test_letter_swap_2(self, overrides=None, exclude=None):
+        assert ca.check_letter_swap("49576", "45976") == True
+
+    def test_letter_swap_3(self, overrides=None, exclude=None):
+        assert ca.check_letter_swap("45967", "45976") == True
+
+    def test_letter_swap_4(self, overrides=None, exclude=None):
+        assert ca.check_letter_swap("47956", "45976") == False
+
+    def test_letter_swap_5(self, overrides=None, exclude=None):
+        assert ca.check_letter_swap("54966", "45976") == False
+
+    def test_letter_swap_6(self, overrides=None, exclude=None):
+        assert ca.check_letter_swap("54967", "45976") == False