The Wonderland Board

Code: Select all

    def parseTopics(self, response):
        soup = BeautifulSoup(response.body)
        # Find topic information
        topics = []
        for link, profileBit in zip(soup.find_all("a", attrs={"class": "topictitle"}),
            soup.find_all("span", attrs={"class": "name"})):
            if link["href"].split("=")[1] in self.topics_to_ignore: # Make sure Announcements and Stickies are not scanned twice while making sure they do get scanned at least once
                continue
            aTopic = Topic()
            aTopic["forumID"] = response.meta["forumid"]
            aTopic["topicID"] = link["href"].split("=")[1]
            aTopic["topicName"] = link.string
            aTopic["posterID"] = profileBit.a["href"].split("=")[2]
            topics.append(aTopic)
            if link.previous_sibling().string in ["Announcement:", "Sticky:"]: # We've scanned this topic before, let's skip it in the future
                self.topics_to_ignore.append(link["href"].split("=")[1])
        # Figure out if there's tomorrow
        hasMultiplePages = soup.find_all("td", align="right", valign="bottom", nowrap="nowrap")
        if hasMultiplePages:
            hasNextPage = hasMultiplePages[0].find_all("a", text="Next")
            if hasNextPage:
                yield Request((self.root_domain + "/" + hasNextPage[0]["href"]), callback=self.parseTopics, meta={"forumid": response.meta["forumid"]})
        for t in topics:
            if t["topicID"] not in ignoreList:
                yield Request((self.root_domain + "/" + "viewtopic.php?t=" + t["topicID"]), callback=self.parsePosts, meta={"topicID": t["topicID"]})

Code: Select all

    def parsePosts(self, response):
        """
        Parse post content.
        """
        soup = BeautifulSoup(response.body)
        def aName(tag):
            return tag.name == "a" and isinstance(tag["name"], int)
        def aHref(tag):
            return tag.name == "a" and tag["href"].startswith("profile.php?mode=viewprofile&u=")
        def spanClass(tag):
            return tag.name == "span" and tag["class"] == "postdetails" and tag.string.startswith("Posted: ")
        def spanClassPostBody(tag):
            return tag.name == "span" and \
                   tag["class"] == "postbody" and not \
                   tag.string.startswith("<br />_________________<br />")
        def attachURL(tag):
            return tag.name == "a" and tag["href"].startswith("download.php?id=")

        def determineContentFetchMode(postid):
            if soup.find("a", href="posting.php?mode=editpost&p=" + postid):
                return "edit"
            elif soup.find("a", href="posting.php?mode=quote&p=" + postid):
                return "quote"
            else:
                return "raw"

        # Find posts information
        posts = []
        attachments = []
        for (pid, userid, username, posttime, content) in zip(
            soup.find_all(aName), # Post ID
            soup.find_all(aHref), # User ID
            soup.find_all("span", attrs={"class": "name"}).b.string,
            soup.find_all(spanClass),
            soup.find_all(spanClassPostBody), # Post body
        ):
            aPost = Post()
            aPost["postID"] = pid
            aPost["topicID"] = response.meta["topicID"]
            aPost["posterID"] = userid.strip("profile.php?mode=viewprofile&u=")
            aPost["postTime"] = posttime.strip("Posted: ")[0:9] # Timestamps are always 10 digits
            attachTable = content.find_next_sibling("table", attrs={"class": "attachtable"})
            # Attachment?
            if attachTable:
                anAttachment = Attachment()
                anAttachment["postID"] = pid
                anAttachment["originalFilename"] = attachTable.find("span", attrs={"class": "gen"}) # The original name
                anAttachment["displayFilename"] = attachTable.find(attachURL)
            # Initiate Post content scraping
            mode = determineContentFetchMode(pid)
            if mode != "raw":
                yield Request((self.root_domain + "/" + "posting.php?mode=" +
                               ("editpost" if p["content"][0] == "edit" else "quote") +
                               "&p=" + p["postID"]),
                    callback=self.parsePostContent,
                    meta={"postID": pid, "content": None, "mode": mode})
            # Initiate User scraping
            if username not in self.users_scanned:
                yield Request((self.root_domain + "/" + userid),
                    callback=self.parseUser)
            posts.append(aPost)
        yield posts
        # Figure out if there's tomorrow
        hasMultiplePages = soup.find("td", align="left", valign="bottom", colspan=2)
        if hasMultiplePages:
            hasNextPage = hasMultiplePages.find("a", text="Next")
            if hasNextPage:
                yield Request((self.root_domain + "/" + hasNextPage["href"]),
                callback=self.parseTopics,
                meta={"topicID": response.meta["topicID"]})