Project: Training a Basic Chatbot Part 2

from chatterbot import ChatBot
import urllib
from bs4 import BeautifulSoup
import re

# sets up the chatbot Noob
chatbot = ChatBot("Noob", logging=False)

wanttotrain = 1

if wanttotrain == True:

    # A regular expression that targets "A: (blah blah) \n" OR "B: (blah blah) \n"
    filt = re.compile('[A|B]: (.*?)\n')
    # A regular expression that targets the reversed URL to get the base URL
    linkfilt = re.compile('.*?(\/.*)')
    links = []

    # looks at the main page and gets a list of links to more links
    main = 'https://www.agendaweb.org/listening/practical-english-conversations.html'
    page = urllib.urlopen(main).read()
    soup = BeautifulSoup(page)
    main = []

    for m in [soup.find_all('a')[9],soup.find_all('a')[16],soup.find_all('a')[21],soup.find_all('a')[18],soup.find_all('a')[22]]:
        main.append(m['href'])

    # uses those links to find the actual links to the dialogue
    for m in main:
        print m
        link = m
        page = urllib.urlopen(link).read()
        soup = BeautifulSoup(page)
        # adds those links onto a list called links
        for l in soup.find_all('a')[:-1]:
            links.append((m, l['href']))

    # a function that extracts the conversation into a list
    def getconvo(url):
        page = urllib.urlopen(url).read()
        soup = BeautifulSoup(page)
        text = soup.get_text()

        return filt.findall(text)
    
    print links
    
    counter = 1
    for l in links:
        # reverses it so that the Regular Expression can be used
        reversedbase = linkfilt.findall(l[0][::-1])
        # reverses it again
        base = reversedbase[0][::-1]
        result = getconvo(base + l[1])
        for x in result:
            print counter, x
            counter += 1
        chatbot.train(result)

while True:
    response = chatbot.get_response(raw_input())
    print response