Web Scraping Basics

pip install requests beautifulsoup4

<html><body>
  <h1>Sharma Sweets – Price List</h1>
  <table id="prices">
    <tr><th>Sweet</th><th>Price</th><th>Stock</th></tr>
    <tr class="item"><td>Kaju Katli</td><td>₹900/kg</td><td>Yes</td></tr>
    <tr class="item"><td>Ghewar</td><td>₹650/kg</td><td>Yes</td></tr>
    <tr class="item sold"><td>Kalakand</td><td>₹600/kg</td><td>No</td></tr>
  </table>
  <a href="/order">Order online</a> <a href="https://wa.me/919829012345">WhatsApp</a>
</body></html>

from bs4 import BeautifulSoup

with open("price_list.html", encoding="utf-8") as f:
    soup = BeautifulSoup(f.read(), "html.parser")

print(soup.h1.get_text(strip=True))              # first <h1>
print(soup.find("td").text)                      # first <td>
print(len(soup.find_all("tr", class_="item")), "items")

for row in soup.find_all("tr", class_="item"):
    name, price, stock = [td.get_text(strip=True) for td in row.find_all("td")]
    print(f"{name:<11} {price:<9} in stock: {stock}")

from bs4 import BeautifulSoup

soup = BeautifulSoup(open("price_list.html", encoding="utf-8").read(), "html.parser")

print([td.text for td in soup.select("#prices tr.item td:first-child")])
print(soup.select_one("tr.sold td").text, "is sold out")
for a in soup.select("a[href]"):
    print(a.text, "→", a["href"])

import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

URL = "https://codingeasily.in/tutorial/python/"
headers = {"User-Agent": "StudentScraper/1.0 (learning project)"}

r = requests.get(URL, headers=headers, timeout=10)
r.raise_for_status()
soup = BeautifulSoup(r.text, "html.parser")

links = soup.select("a.topic-card-link")
print("lessons found:", len(links))
for a in links[:5]:
    title = a.select_one(".name.english-content").get_text(strip=True)
    print("-", title, "→", urljoin(URL, a["href"]))

from urllib import robotparser

rp = robotparser.RobotFileParser()
rp.set_url("https://codingeasily.in/robots.txt")
rp.read()

for path in ["/tutorial/python/", "/search/", "/config/"]:
    allowed = rp.can_fetch("StudentScraper", "https://codingeasily.in" + path)
    print(f"{path:<20} allowed: {allowed}")
print("crawl delay:", rp.crawl_delay("StudentScraper"), "second(s)")

import csv, time
import requests
from bs4 import BeautifulSoup

BASE = "https://codingeasily.in"
pages = ["/blog/"]                      # add more pages to this list
rows = []
for path in pages:
    html = requests.get(BASE + path, timeout=10, headers={"User-Agent": "StudentScraper/1.0"}).text
    soup = BeautifulSoup(html, "html.parser")
    for h3 in soup.select("h3[data-en]"):
        rows.append({"title": h3["data-en"], "title_hi": h3["data-hi"]})
    time.sleep(1)                       # be polite: 1 request per second

with open("blog_titles.csv", "w", newline="", encoding="utf-8-sig") as f:
    w = csv.DictWriter(f, fieldnames=["title", "title_hi"])
    w.writeheader()
    w.writerows(rows)

print(len(rows), "titles saved. First two:")
for r in rows[:2]:
    print("-", r["title"], "|", r["title_hi"])

import pandas as pd
from io import StringIO

html = open("price_list.html", encoding="utf-8").read()
table = pd.read_html(StringIO(html), attrs={"id": "prices"})[0]
table["Price (₹)"] = table["Price"].str.extract(r"(\d+)").astype(int)
print(table[["Sweet", "Price (₹)", "Stock"]])