Add topic listing support

.gitignore

@@ -2,3 +2,4 @@ __pycache__/
 tildee.egg-info/
 dist/
 docs/_build
+docs/.mypy_cache

docs/client.rst

@@ -44,6 +44,15 @@ If you want to use data from comments or topics, you can use Tildee to fetch the
 .. autofunction:: tildee.TildesClient.fetch_comment
 
 
+Fetching topic listings
+-----------------------
+
+If you either want to fetch topic listings for tags and/or groups or search for a phrase, use the ``fetch_filtered_topic_listing`` or the ``fetch_search_topic_listing`` method. Note that they only return the limited data the topic listing page offers.
+
+.. autofunction:: tildee.TildesClient.fetch_filtered_topic_listing
+
+.. autofunction:: tildee.TildesClient.fetch_search_topic_listing
+
 Interacting with topics
 -----------------------
 

docs/conf.py

@@ -15,7 +15,7 @@ import sys
 
 sys.path.insert(0, os.path.abspath(".."))
 from tildee import *
-
+from tildee.models import *
 
 # -- Project information -----------------------------------------------------
 

docs/models.rst

@@ -3,27 +3,30 @@ Model Classes
 
 These are all representations of "things" on Tildes, created from HTML from the site and provide easy access to the interesting parts of that HTML.
 
-.. autoclass:: tildee.TildesTopic
+.. autoclass:: tildee.models.TildesTopic
    :members:
 
-.. autoclass:: tildee.TildesComment
+.. autoclass:: tildee.models.TildesPartialTopic
    :members:
 
-.. autoclass:: tildee.TildesTopicLogEntry
+.. autoclass:: tildee.models.TildesComment
    :members:
 
-.. autoclass:: tildee.TildesTopicLogEntryKind
+.. autoclass:: tildee.models.TildesTopicLogEntry
    :members:
 
-.. autoclass:: tildee.TildesNotification
+.. autoclass:: tildee.models.TildesTopicLogEntryKind
+   :members:
+
+.. autoclass:: tildee.models.TildesNotification
 	 :members:
 
-.. autoclass:: tildee.TildesNotificationKind
+.. autoclass:: tildee.models.TildesNotificationKind
 	 :members:
 	 :undoc-members:
 
-.. autoclass:: tildee.TildesConversation
+.. autoclass:: tildee.models.TildesConversation
 	 :members:
 
-.. autoclass:: tildee.TildesMessage
+.. autoclass:: tildee.models.TildesMessage
 	 :members:

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tildee"
-version = "0.2.4"
+version = "0.2.5"
 description = "A client for tildes.net"
 license = "MIT"
 readme = "README.md"

tildee/__init__.py

@@ -8,6 +8,7 @@ from tildee.models import (
     TildesComment,
     TildesNotification,
     TildesConversation,
+    TildesPartialTopic,
 )
 from typing import Union, List, Optional
 
@@ -174,6 +175,72 @@ class TildesClient:
         r = self._get(f"/~group_name_here/{topic_id36}")
         return TildesTopic(r.text)
 
+    def fetch_topic_listing(self, path: str) -> List[TildesPartialTopic]:
+        """Fetches, parses and returns all topics from a certain URL.
+
+        I.e. an empty string for the home page, ``~group`` for a group or ``search?q=a`` for a search.
+        Appends ``per_page=100`` to the path for maximum results.
+
+        :param str path: The URL to fetch from.
+        :rtype: List[TildesPartialTopic]
+        :return: The requested topics."""
+        r = None
+        if "?" in path:
+            r = self._get(f"/{path}&per_page=100")
+        else:
+            r = self._get(f"/{path}?per_page=100")
+
+        articles = html.fromstring(r.text).cssselect("article.topic")
+        topics = []
+        for article in articles:
+            topics.append(TildesPartialTopic(etree.tostring(article)))
+
+        return topics
+
+    def fetch_filtered_topic_listing(self, group: str = "", tag: str = "", **kwargs):
+        """Fetches a filtered list of topics. Automatically adds ``per_page=100`` to the query string.
+
+        :param str group: The group to filter for. Leave empty to search all subscribed groups.
+        :param str tag: The tag to filter for, Tildes currently only supports filtering for one tag.
+        :rtype: List[TildesPartialTopic]
+        :return: Up to 100 topics matching the filters."""
+        query_str: str = ""
+        if group:
+            query_str += f"~{group}/"
+        if tag:
+            query_str += f"?tag={tag.replace(' ', '_')}"
+        if kwargs:
+            if tag:
+                query_str += "&"
+            else:
+                query_str += "?"
+
+            for k, v in kwargs.items():
+                query_str += f"{k}={v}&"
+
+            query_str = query_str[:-1]
+
+        return self.fetch_topic_listing(query_str)
+
+    def fetch_search_topic_listing(self, query: str = "", **kwargs):
+        """Fetches a search result's list of topics. Automatically adds ``per_page=100`` to the query string.
+
+        :param str query: The string to search for.
+        :rtype: List[TildesPartialTopic]
+        :return: Up to 100 topics matching the query string."""
+        if not query:
+            raise RuntimeError("Query string must not be empty.")
+        query_str: str = f"search?q={query}"
+        if kwargs:
+            query_str += "&"
+
+            for k, v in kwargs.items():
+                query_str += f"{k}={v}&"
+
+            query_str = query_str[:-1]
+
+        return self.fetch_topic_listing(query_str)
+
     def fetch_comment(self, comment_id36: str) -> TildesComment:
         """Fetches, parses and returns a single comment as an object for further processing.
 

tildee/models.py

@@ -67,6 +67,58 @@ class TildesTopic:
             self.comments.append(TildesComment(etree.tostring(comment)))
 
 
+class TildesPartialTopic:
+    """Represents a topic on Tildes as fetched from the topic listings, generated from its surrounding ``<article>`` tag. Has only reduced information available.
+
+    :ivar str id36: The topic's id36.
+    :ivar str title: The topic's title.
+    :ivar Optional[str] group: The topic's group, if provided in the listing.
+    :ivar str author: The topic's author.
+    :ivar Optional[str] link: The topic's link, if available.
+    :ivar Optional[str] content_html: The text of this topic as rendered by the site, if available.
+    :ivar int num_votes: The amount of votes this topic has received.
+    :ivar int num_comments: The amount of comments on this topic.
+    :ivar List[str] tags: The tags on this topics.
+    :ivar str timestamp: The topic's timestamp."""
+
+    def __init__(self, text):
+        self._tree = html.fromstring(text)
+        self.id36 = self._tree.cssselect("article")[0].attrib["id"].split("-")[1]
+        self.title = self._tree.cssselect("h1.topic-title > a")[0].text
+        self.group = None
+        try:
+            self.group = self._tree.cssselect(".topic-group > a")[0].text[1:]
+        except IndexError:
+            pass
+
+        self.author = self._tree.cssselect("article")[0].attrib["data-topic-posted-by"]
+        self.link = None
+        self.content_html = None
+        if not self._tree.cssselect("header h1 > a")[0].attrib["href"].startswith("/"):
+            self.link = self._tree.cssselect("header h1 > a")[0].attrib["href"]
+        elif self._tree.cssselect(".topic-text-excerpt"):
+            tree = self._tree.cssselect(".topic-text-excerpt")[0]
+            etree.strip_elements(tree, "summary")
+            self.content_html = str(etree.tostring(tree))
+        try:
+            self.num_votes = int(
+                self._tree.cssselect("span.topic-voting-votes")[0].text
+            )
+        except IndexError:
+            self.num_votes = 0
+        self.num_comments = int(
+            re.findall(
+                "[0-9]+", self._tree.cssselect(".topic-info-comments > a")[0].text
+            )[0]
+        )
+
+        self.tags = []
+        for element in self._tree.cssselect("ul.topic-tags > li > a"):
+            self.tags.append(element.text)
+
+        self.timestamp = self._tree.cssselect("time")[0].attrib["datetime"]
+
+
 class TildesComment:
     """Represents a single comment on Tildes, generated from its surrounding ``<article>`` tag.