Reorganized chat downloader

clipper/api.py

@@ -31,7 +31,7 @@ class TwitchApi:
     def get_user_status(self, streamer):
         try:
             streams = self.twitch.get_streams(user_login=streamer)
-            if streams is None or len(streams) < 1:
+            if streams is None or len(streams["data"]) < 1:
                 return TwitchStreamStatus.OFFLINE
             else:
                 return TwitchStreamStatus.ONLINE
@@ -47,7 +47,7 @@ class TwitchApi:
 
     def get_user_chat_channel(self, streamer_name):
         streams = self.twitch.get_streams(user_login=streamer_name)
-        if streams is None or len(streams) < 1:
+        if streams is None or len(streams["data"]) < 1:
             return None
         return streams["data"][0]["user_login"]
 
@@ -66,7 +66,8 @@ class ChatConnection:
         # Need to verify channel name.. case sensitive
         channel = self.api.get_user_chat_channel(self.streamer_name)
         if not channel:
-            logger.error("Cannot find streamer channel")
+            logger.error("Cannot find streamer channel, Offline?")
+            return
 
         self.connect_to_chat(f"#{channel}")
 
@@ -75,13 +76,14 @@ class ChatConnection:
         self.connection.connect((TW_CHAT_SERVER, TW_CHAT_PORT))
         # public data to join hat
         self.connection.send(f"PASS couldBeRandomString\n".encode("utf-8"))
-        self.connection.send(f"NICK justinfan123\n".encode("utf-8"))
+        self.connection.send(f"NICK justinfan113\n".encode("utf-8"))
         self.connection.send(f"JOIN {channel}\n".encode("utf-8"))
 
+        logger.info("Connected to %s", channel)
+
         try:
             while True:
-                msg = self.connection.recv(4096).decode('utf-8')
+                msg = self.connection.recv(8192).decode('utf-8')
-                logger.warning(f"Twitch message->  {msg}")
                 if self.on_message:
                     self.on_message(msg)
         except BaseException as e:

clipper/chat.py

@@ -1,40 +1,35 @@
 import logging
-import os
+from datetime import datetime
 
 logger = logging.getLogger(__name__)
 
-
+CHAT_DIVIDER = "<~|~>"
-def parse_msg(msg):
-    """Breaks a message from an IRC server into its prefix, command, and arguments.
-    """
-    prefix = ''
-    trailing = []
-    if not msg:
-        raise ValueError("Empty line.")
-
-    if msg[0] == ':':
-        prefix, msg = msg[1:].split(' ', 1)
-    if msg.find(' :') != -1:
-        msg, trailing = msg.split(' :', 1)
-        args = msg.split()
-        args.append(trailing)
-    else:
-        args = msg.split()
-    command = args.pop(0)
-    return prefix, command, args
 
 
 class TwitchChatRecorder:
-    def __init__(self, api, streamer_name, recording_folder):
+    is_running = False
-        self.recording_folder = recording_folder
+
-        self.streamer_name = streamer_name
+    def __init__(self, api, debug=False):
+        self.debug = debug
         self.api = api
 
-    def run(self, file_template):
+    def run(self, streamer_name, output_file):
-        file_name = os.path.join(self.recording_folder, f"{file_template}.txt", )
+        with open(output_file, "w") as stream:
-        with open(file_name, "w") as stream:
             def on_message(twitch_msg):
-                prefix, command, args = parse_msg(twitch_msg)
+                user, msg = self.parse_msg(twitch_msg)
-                stream.writelines(str(args))
+                if msg:
+                    msg_line = f"{str(datetime.now())}{CHAT_DIVIDER}{user}{CHAT_DIVIDER}{msg}"
+                    stream.write(msg_line)
+                    stream.flush()
 
-            self.api.start_chat(self.streamer_name, on_message)
+                    if self.debug:
+                        logger.info("Chat: %s", msg_line)
+
+            self.is_running = True
+            self.api.start_chat(streamer_name, on_message)
+
+    def parse_msg(self, msg):
+        try:
+            return msg[1:].split('!')[0], msg.split(":", 2)[2]
+        except BaseException as e:
+            return None, None

clipper/recorder.py

@@ -2,6 +2,7 @@ import logging
 import os
 import time
 import sys
+import threading
 from datetime import datetime
 
 from clipper.api import TwitchApi, TwitchStreamStatus
@@ -27,24 +28,26 @@ class Recorder:
     def __init__(self, config):
         self.config = config
         self.api = TwitchApi(config.tw_client, config.tw_secret)
-        self.recording_folder = os.path.join(self.config.output_folder, self.config.tw_streamer)
+        self.streamer_folder = os.path.join(self.config.output_folder, self.config.tw_streamer)
-        self.video_recorder = TwitchVideoRecorder(self.api, config.tw_streamer, self.recording_folder)
+        self.video_recorder = TwitchVideoRecorder()
-        self.chat_recorder = TwitchChatRecorder(self.api, config.tw_streamer, self.recording_folder)
+        self.chat_recorder = TwitchChatRecorder(self.api, debug=True)
 
     def run(self):
-        if os.path.isdir(self.recording_folder) is False:
-            logger.info("Recording folder `%s` does not exists. Create it", self.recording_folder)
-            os.makedirs(self.recording_folder)
-
         while True:
             logger.info("Start watching streamer %s", self.config.tw_streamer)
             status = self.api.get_user_status(self.config.tw_streamer)
             if status == TwitchStreamStatus.ONLINE:
                 logger.info("Streamer %s is online. Start recording", self.config.tw_streamer)
 
-                now = datetime.now()
+                start_time = datetime.now()
-                file_template = "{0}-{1}".format(self.config.tw_streamer, now.strftime("%H-%M-%S"))
+                record_folder_name = start_time.strftime("%d-%m-%Y_%H-%M-%S")
-                self.chat_recorder.run(file_template)
+                record_folder = os.path.join(self.streamer_folder, record_folder_name)
+                os.makedirs(record_folder)
+
+                chat_file = os.path.join(record_folder,  "chat.txt")
+                video_file = os.path.join(record_folder, "video.mp4")
+
+                self.chat_recorder.run(self.config.tw_streamer, chat_file)
                 # self.video_recorder.run(file_template)
 
                 logger.info("Streamer %s has finished stream", self.config.tw_streamer)

clipper/video.py

@@ -1,51 +1,30 @@
-import datetime
 import logging
-import os
 import subprocess
 
 logger = logging.getLogger(__name__)
 
 
 class TwitchVideoRecorder:
-    access_token = None
+    is_running = False
+    refresh_timeout = 15
+    streamlink_process = None
 
-    def __init__(self, authenticator, streamer_name, recording_folder, quality="480p", on_finish=None):
+    def run(self, streamer_name, output_file, quality="480p"):
-        # global configuration
+        self._record_stream(streamer_name, output_file, quality)
-        self.disable_ffmpeg = False
-        self.refresh_timeout = 15
-        self.recording_folder = recording_folder
-        self.stream_uid = None
-        self.on_finish = on_finish
 
-        # twitch configuration
+    def stop(self):
-        self.streamer_name = streamer_name
+        if self.streamlink_process:
-        self.quality = quality
+            self.streamlink_process.kill()
-        self.authenticator = authenticator
 
-    def run(self):
+    def _record_stream(self, streamer_name, output_file, quality):
-        # make sure the interval to check user availability is not less than 15 seconds
+        # subprocess.call()
-        if self.refresh_timeout < 15:
+        self.streamlink_process = subprocess.Popen([
-            logger.warning("check interval should not be lower than 15 seconds")
-            self.refresh_timeout = 15
-            logger.warning("check interval set to 15 seconds")
-
-        self.record_stream(self.streamer_name, recording_path)
-
-    def record_stream(self, channel, recording_path):
-        filename = self.streamer_name + " - " + datetime.datetime.now() \
-            .strftime("%Y-%m-%d %Hh%Mm%Ss") + " - " + channel.get("title") + ".mp4"
-
-        filename = "".join(x for x in filename if x.isalnum() or x in [" ", "-", "_", "."])
-        recorded_filename = os.path.join(recording_path, filename)
-
-        # start streamlink process
-        subprocess.call([
             "streamlink",
             "--twitch-disable-ads",
-            "twitch.tv/" + self.streamer_name,
+            "twitch.tv/" + streamer_name,
-            self.quality,
+            quality,
             "-o",
-            recorded_filename
+            output_file
         ])
 
-        return recorded_filename
+        self.is_running = True