feat: Add native Android driver tracking app and real-time backend/frontend bus tracking system
This commit is contained in:
Shanmuga Krishnan S M
203
telegram-bot/download_and_scrape_maps.py
Normal file
203
telegram-bot/download_and_scrape_maps.py
Normal file
@@ -0,0 +1,203 @@
|
||||
"""
|
||||
Download all RIT Transport map pages (map00.php - map45.php) live,
|
||||
parse their stops and polyline coordinates, and output map_coordinates.json.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import time
|
||||
import requests
|
||||
|
||||
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
MAP_PAGES_DIR = os.path.join(CURRENT_DIR, "map_pages")
|
||||
MAP_COORDS_OUTPUT = os.path.join(CURRENT_DIR, "map_coordinates.json")
|
||||
|
||||
os.makedirs(MAP_PAGES_DIR, exist_ok=True)
|
||||
|
||||
|
||||
def download_maps():
|
||||
print("Downloading RIT transport map pages live from fit25.com...")
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
}
|
||||
|
||||
downloaded_count = 0
|
||||
# Try downloading maps from 00 to 45
|
||||
for i in range(46):
|
||||
url = f"https://fit25.com/ritRouteMap/map{i:02d}.php?cb={int(time.time())}"
|
||||
filepath = os.path.join(MAP_PAGES_DIR, f"map{i:02d}.html")
|
||||
try:
|
||||
print(f"Fetching Map {i:02d}: {url} ...")
|
||||
res = requests.get(url, headers=headers, timeout=15)
|
||||
if res.status_code == 200:
|
||||
html = res.text
|
||||
if "newpoints" in html or "GPolyline" in html:
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
f.write(html)
|
||||
print(f" -> Saved {len(html)} bytes to {os.path.basename(filepath)}")
|
||||
downloaded_count += 1
|
||||
else:
|
||||
print(f" -> Skipped (no map data found in response)")
|
||||
else:
|
||||
print(f" -> Skipped (status code {res.status_code})")
|
||||
|
||||
# Respectful delay
|
||||
time.sleep(0.5)
|
||||
except Exception as e:
|
||||
print(f" -> Error fetching map {i:02d}: {e}")
|
||||
time.sleep(1.0)
|
||||
|
||||
print(f"Downloaded {downloaded_count} map page HTML files.")
|
||||
|
||||
|
||||
def extract_stops(html_text: str) -> list:
|
||||
stops = []
|
||||
# Regex to capture newpoints entries, handling potential whitespace variation
|
||||
pattern = re.compile(
|
||||
r"new\s+Array\s*\(\s*"
|
||||
r"([\d.]+)\s*,\s*([\d.]+)\s*,"
|
||||
r"\s*[^,]+\s*,"
|
||||
r"\s*[^,]+\s*,"
|
||||
r"\s*'([^']*)'"
|
||||
r"\s*\)",
|
||||
re.IGNORECASE
|
||||
)
|
||||
for m in pattern.finditer(html_text):
|
||||
try:
|
||||
lat = float(m.group(1))
|
||||
lng = float(m.group(2))
|
||||
label = m.group(3).strip()
|
||||
|
||||
# Match formats like:
|
||||
# - 'Route no:1 - Lift Gate'
|
||||
# - 'Route no:9Vyasarpadi'
|
||||
# - 'Route no: 9MKB Nagar'
|
||||
# - 'Route no:13 - ICF'
|
||||
# - 'Route no:14A - Kakallur'
|
||||
label_match = re.search(r"Route\s+no\s*:\s*([a-zA-Z0-9]+)\s*-?\s*(.*)", label, re.IGNORECASE)
|
||||
if label_match:
|
||||
route_num = label_match.group(1).strip()
|
||||
stop_name = label_match.group(2).strip()
|
||||
# Clean up multiple spaces or hyphens in stop name
|
||||
stop_name = re.sub(r'^-?\s*', '', stop_name)
|
||||
stop_name = " ".join(stop_name.split())
|
||||
if stop_name:
|
||||
stops.append({"route_num": route_num, "name": stop_name, "lat": lat, "lng": lng})
|
||||
except Exception as e:
|
||||
print(f" Error parsing stop line: {m.group(0)} - {e}")
|
||||
return stops
|
||||
|
||||
|
||||
def extract_polylines(html_text: str) -> list:
|
||||
polylines = []
|
||||
poly_pattern = re.compile(r"GPolyline\s*\(\s*\[(.*?)\]", re.DOTALL | re.IGNORECASE)
|
||||
coord_pattern = re.compile(r"GLatLng\s*\(\s*([\d.]+)\s*,\s*([\d.]+)\s*\)")
|
||||
for poly_match in poly_pattern.finditer(html_text):
|
||||
block = poly_match.group(1)
|
||||
coords = []
|
||||
for coord_match in coord_pattern.finditer(block):
|
||||
coords.append([float(coord_match.group(1)), float(coord_match.group(2))])
|
||||
if coords:
|
||||
polylines.append(coords)
|
||||
return polylines
|
||||
|
||||
|
||||
def merge_polylines(polylines):
|
||||
if not polylines:
|
||||
return []
|
||||
merged = list(polylines[0])
|
||||
for seg in polylines[1:]:
|
||||
if not seg:
|
||||
continue
|
||||
if merged and seg:
|
||||
last = merged[-1]
|
||||
first = seg[0]
|
||||
# Connect if close, else just extend
|
||||
if abs(last[0] - first[0]) < 0.001 and abs(last[1] - first[1]) < 0.001:
|
||||
seg = seg[1:]
|
||||
merged.extend(seg)
|
||||
return merged
|
||||
|
||||
|
||||
def parse_and_save_coordinates():
|
||||
print("\nParsing downloaded map HTML files...")
|
||||
all_routes = {}
|
||||
|
||||
html_files = sorted([f for f in os.listdir(MAP_PAGES_DIR) if f.endswith(".html")])
|
||||
if not html_files:
|
||||
print("Error: No downloaded HTML files found in map_pages directory!")
|
||||
return
|
||||
|
||||
for filename in html_files:
|
||||
filepath = os.path.join(MAP_PAGES_DIR, filename)
|
||||
print(f"Parsing {filename} ...")
|
||||
with open(filepath, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
|
||||
stops = extract_stops(content)
|
||||
polylines = extract_polylines(content)
|
||||
merged_poly = merge_polylines(polylines)
|
||||
|
||||
if not stops and not polylines:
|
||||
print(f" -> No data parsed")
|
||||
continue
|
||||
|
||||
route_nums = set()
|
||||
for s in stops:
|
||||
rn = s["route_num"]
|
||||
# Clean route num (e.g. '01' -> '1', '14A' -> '14A')
|
||||
rn_clean = re.sub(r'^0+', '', rn)
|
||||
route_nums.add(rn_clean)
|
||||
key = f"R{rn_clean}"
|
||||
if key not in all_routes:
|
||||
all_routes[key] = {"stops": [], "polyline": []}
|
||||
all_routes[key]["stops"].append({
|
||||
"name": s["name"],
|
||||
"lat": s["lat"],
|
||||
"lng": s["lng"]
|
||||
})
|
||||
|
||||
if merged_poly:
|
||||
# If stops are all for a single route, assign the polyline to it
|
||||
if len(route_nums) == 1:
|
||||
rn = list(route_nums)[0]
|
||||
key = f"R{rn}"
|
||||
existing = all_routes.get(key, {}).get("polyline", [])
|
||||
if existing:
|
||||
all_routes[key]["polyline"] = merge_polylines([existing, merged_poly])
|
||||
else:
|
||||
all_routes[key]["polyline"] = merged_poly
|
||||
# Otherwise, try parsing route number from polyline variable comments or assign to all routes found
|
||||
else:
|
||||
# Fallback: assign polyline to all routes mentioned in this file
|
||||
for rn in sorted(route_nums):
|
||||
key = f"R{rn}"
|
||||
if not all_routes[key].get("polyline"):
|
||||
all_routes[key]["polyline"] = merged_poly
|
||||
|
||||
print(f" -> Found {len(stops)} stops, {len(polylines)} polyline segments. Associated routes: {sorted(route_nums)}")
|
||||
|
||||
# Deduplicate stops for each route
|
||||
for key, data in all_routes.items():
|
||||
seen = set()
|
||||
unique = []
|
||||
for s in data["stops"]:
|
||||
ident = (s["name"].lower().strip(), round(s["lat"], 5), round(s["lng"], 5))
|
||||
if ident not in seen:
|
||||
seen.add(ident)
|
||||
unique.append(s)
|
||||
data["stops"] = unique
|
||||
|
||||
with open(MAP_COORDS_OUTPUT, "w", encoding="utf-8") as f:
|
||||
json.dump(all_routes, f, indent=2, ensure_ascii=False)
|
||||
|
||||
total_stops = sum(len(d["stops"]) for d in all_routes.values())
|
||||
total_wp = sum(len(d["polyline"]) for d in all_routes.values())
|
||||
print(f"\nDone! Scraped {len(all_routes)} routes, {total_stops} stops, {total_wp} polyline waypoints.")
|
||||
print(f"Output saved to: {MAP_COORDS_OUTPUT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
download_maps()
|
||||
parse_and_save_coordinates()
|
||||
Reference in New Issue
Block a user