python之如何使用Python获取域中的所有链接

我想使用 Python 获取域中给定“根”URL(在列表中)的所有链接。假设给定一个 URL http://www.example.com这应该返回此页面上与根 URL 相同域的所有链接，然后递归访问这些链接中的每一个并提取相同域的所有链接等等。我所说的相同域的意思是如果给出 http://www.example.com我想要返回的唯一链接是 http://www.example.com/something , http://www.example.com/somethingelse ... 任何外部的东西，例如 http://www.otherwebsite.com应该被丢弃。我如何使用 Python 执行此操作？

编辑:我尝试使用 lxml。我不认为这完全有效，我不确定如何考虑到已处理页面的链接(导致无限循环)。

import urllib 
import lxml.html 
 
#given a url returns list of all sublinks within the same domain 
def getLinks(url): 
        urlList = [] 
        urlList.append(url) 
        sublinks = getSubLinks(url) 
        for link in sublinks: 
                absolute = url+'/'+link 
                urlList.extend(getLinks(absolute)) 
         return urlList 
 
#determine whether two links are within the same domain 
def sameDomain(url, dom): 
    return url.startswith(dom) 
 
#get tree of sublinks in same domain, url is root 
def getSubLinks(url): 
    sublinks = [] 
    connection = urllib.urlopen(url) 
    dom = lxml.html.fromstring(connection.read()) 
    for link in dom.xpath('//a/@href'): 
                if not (link.startswith('#') or link.startswith('http') or link.startswith('mailto:')): 
                        sublinks.append(link) 
    return sublinks 

~

请您参考如下方法:

import sys 
import requests 
import hashlib 
from bs4 import BeautifulSoup 
from datetime import datetime 
 
def get_soup(link): 
    """ 
    Return the BeautifulSoup object for input link 
    """ 
    request_object = requests.get(link, auth=('user', 'pass')) 
    soup = BeautifulSoup(request_object.content) 
    return soup 
 
def get_status_code(link): 
    """ 
    Return the error code for any url 
    param: link 
    """ 
    try: 
        error_code = requests.get(link).status_code 
    except requests.exceptions.ConnectionError: 
        error_code =  
    return error_code 
 
def find_internal_urls(lufthansa_url, depth=0, max_depth=2): 
    all_urls_info = [] 
    status_dict = {} 
    soup = get_soup(lufthansa_url) 
    a_tags = soup.findAll("a", href=True) 
 
    if depth > max_depth: 
        return {} 
    else: 
        for a_tag in a_tags: 
            if "http" not in a_tag["href"] and "/" in a_tag["href"]: 
                url = "http://www.lufthansa.com" + a_tag['href'] 
            elif "http" in a_tag["href"]: 
                url = a_tag["href"] 
            else: 
                continue 
            status_dict["url"] = url 
            status_dict["status_code"] = get_status_code(url) 
            status_dict["timestamp"] = datetime.now() 
            status_dict["depth"] = depth + 1 
            all_urls_info.append(status_dict) 
    return all_urls_info 
if __name__ == "__main__": 
    depth = 2 # suppose  
    all_page_urls = find_internal_urls("someurl", 2, 2) 
    if depth > 1: 
        for status_dict in all_page_urls: 
            find_internal_urls(status_dict['url']) 

上面的代码片段包含从 lufthansa arlines 网站抓取 url 的必要模块。这里唯一额外的是您可以指定要递归抓取的深度。