Browse Source

Add topic listing support

master
parent
commit
8e0d4a53fa
No known key found for this signature in database GPG Key ID: DA34C790D267C164
7 changed files with 142 additions and 10 deletions
  1. +1
    -0
      .gitignore
  2. +9
    -0
      docs/client.rst
  3. +1
    -1
      docs/conf.py
  4. +11
    -8
      docs/models.rst
  5. +1
    -1
      pyproject.toml
  6. +67
    -0
      tildee/__init__.py
  7. +52
    -0
      tildee/models.py

+ 1
- 0
.gitignore View File

@ -2,3 +2,4 @@ __pycache__/
tildee.egg-info/
dist/
docs/_build
docs/.mypy_cache

+ 9
- 0
docs/client.rst View File

@ -44,6 +44,15 @@ If you want to use data from comments or topics, you can use Tildee to fetch the
.. autofunction:: tildee.TildesClient.fetch_comment
Fetching topic listings
-----------------------
If you either want to fetch topic listings for tags and/or groups or search for a phrase, use the ``fetch_filtered_topic_listing`` or the ``fetch_search_topic_listing`` method. Note that they only return the limited data the topic listing page offers.
.. autofunction:: tildee.TildesClient.fetch_filtered_topic_listing
.. autofunction:: tildee.TildesClient.fetch_search_topic_listing
Interacting with topics
-----------------------


+ 1
- 1
docs/conf.py View File

@ -15,7 +15,7 @@ import sys
sys.path.insert(0, os.path.abspath(".."))
from tildee import *
from tildee.models import *
# -- Project information -----------------------------------------------------


+ 11
- 8
docs/models.rst View File

@ -3,27 +3,30 @@ Model Classes
These are all representations of "things" on Tildes, created from HTML from the site and provide easy access to the interesting parts of that HTML.
.. autoclass:: tildee.TildesTopic
.. autoclass:: tildee.models.TildesTopic
:members:
.. autoclass:: tildee.TildesComment
.. autoclass:: tildee.models.TildesPartialTopic
:members:
.. autoclass:: tildee.TildesTopicLogEntry
.. autoclass:: tildee.models.TildesComment
:members:
.. autoclass:: tildee.TildesTopicLogEntryKind
.. autoclass:: tildee.models.TildesTopicLogEntry
:members:
.. autoclass:: tildee.TildesNotification
.. autoclass:: tildee.models.TildesTopicLogEntryKind
:members:
.. autoclass:: tildee.models.TildesNotification
:members:
.. autoclass:: tildee.TildesNotificationKind
.. autoclass:: tildee.models.TildesNotificationKind
:members:
:undoc-members:
.. autoclass:: tildee.TildesConversation
.. autoclass:: tildee.models.TildesConversation
:members:
.. autoclass:: tildee.TildesMessage
.. autoclass:: tildee.models.TildesMessage
:members:

+ 1
- 1
pyproject.toml View File

@ -1,6 +1,6 @@
[tool.poetry]
name = "tildee"
version = "0.2.4"
version = "0.2.5"
description = "A client for tildes.net"
license = "MIT"
readme = "README.md"


+ 67
- 0
tildee/__init__.py View File

@ -8,6 +8,7 @@ from tildee.models import (
TildesComment,
TildesNotification,
TildesConversation,
TildesPartialTopic,
)
from typing import Union, List, Optional
@ -174,6 +175,72 @@ class TildesClient:
r = self._get(f"/~group_name_here/{topic_id36}")
return TildesTopic(r.text)
def fetch_topic_listing(self, path: str) -> List[TildesPartialTopic]:
"""Fetches, parses and returns all topics from a certain URL.
I.e. an empty string for the home page, ``~group`` for a group or ``search?q=a`` for a search.
Appends ``per_page=100`` to the path for maximum results.
:param str path: The URL to fetch from.
:rtype: List[TildesPartialTopic]
:return: The requested topics."""
r = None
if "?" in path:
r = self._get(f"/{path}&per_page=100")
else:
r = self._get(f"/{path}?per_page=100")
articles = html.fromstring(r.text).cssselect("article.topic")
topics = []
for article in articles:
topics.append(TildesPartialTopic(etree.tostring(article)))
return topics
def fetch_filtered_topic_listing(self, group: str = "", tag: str = "", **kwargs):
"""Fetches a filtered list of topics. Automatically adds ``per_page=100`` to the query string.
:param str group: The group to filter for. Leave empty to search all subscribed groups.
:param str tag: The tag to filter for, Tildes currently only supports filtering for one tag.
:rtype: List[TildesPartialTopic]
:return: Up to 100 topics matching the filters."""
query_str: str = ""
if group:
query_str += f"~{group}/"
if tag:
query_str += f"?tag={tag.replace(' ', '_')}"
if kwargs:
if tag:
query_str += "&"
else:
query_str += "?"
for k, v in kwargs.items():
query_str += f"{k}={v}&"
query_str = query_str[:-1]
return self.fetch_topic_listing(query_str)
def fetch_search_topic_listing(self, query: str = "", **kwargs):
"""Fetches a search result's list of topics. Automatically adds ``per_page=100`` to the query string.
:param str query: The string to search for.
:rtype: List[TildesPartialTopic]
:return: Up to 100 topics matching the query string."""
if not query:
raise RuntimeError("Query string must not be empty.")
query_str: str = f"search?q={query}"
if kwargs:
query_str += "&"
for k, v in kwargs.items():
query_str += f"{k}={v}&"
query_str = query_str[:-1]
return self.fetch_topic_listing(query_str)
def fetch_comment(self, comment_id36: str) -> TildesComment:
"""Fetches, parses and returns a single comment as an object for further processing.


+ 52
- 0
tildee/models.py View File

@ -67,6 +67,58 @@ class TildesTopic:
self.comments.append(TildesComment(etree.tostring(comment)))
class TildesPartialTopic:
"""Represents a topic on Tildes as fetched from the topic listings, generated from its surrounding ``<article>`` tag. Has only reduced information available.
:ivar str id36: The topic's id36.
:ivar str title: The topic's title.
:ivar Optional[str] group: The topic's group, if provided in the listing.
:ivar str author: The topic's author.
:ivar Optional[str] link: The topic's link, if available.
:ivar Optional[str] content_html: The text of this topic as rendered by the site, if available.
:ivar int num_votes: The amount of votes this topic has received.
:ivar int num_comments: The amount of comments on this topic.
:ivar List[str] tags: The tags on this topics.
:ivar str timestamp: The topic's timestamp."""
def __init__(self, text):
self._tree = html.fromstring(text)
self.id36 = self._tree.cssselect("article")[0].attrib["id"].split("-")[1]
self.title = self._tree.cssselect("h1.topic-title > a")[0].text
self.group = None
try:
self.group = self._tree.cssselect(".topic-group > a")[0].text[1:]
except IndexError:
pass
self.author = self._tree.cssselect("article")[0].attrib["data-topic-posted-by"]
self.link = None
self.content_html = None
if not self._tree.cssselect("header h1 > a")[0].attrib["href"].startswith("/"):
self.link = self._tree.cssselect("header h1 > a")[0].attrib["href"]
elif self._tree.cssselect(".topic-text-excerpt"):
tree = self._tree.cssselect(".topic-text-excerpt")[0]
etree.strip_elements(tree, "summary")
self.content_html = str(etree.tostring(tree))
try:
self.num_votes = int(
self._tree.cssselect("span.topic-voting-votes")[0].text
)
except IndexError:
self.num_votes = 0
self.num_comments = int(
re.findall(
"[0-9]+", self._tree.cssselect(".topic-info-comments > a")[0].text
)[0]
)
self.tags = []
for element in self._tree.cssselect("ul.topic-tags > li > a"):
self.tags.append(element.text)
self.timestamp = self._tree.cssselect("time")[0].attrib["datetime"]
class TildesComment:
"""Represents a single comment on Tildes, generated from its surrounding ``<article>`` tag.


Loading…
Cancel
Save