204 lines
7.4 KiB
Python
204 lines
7.4 KiB
Python
"""
|
|
Download all RIT Transport map pages (map00.php - map45.php) live,
|
|
parse their stops and polyline coordinates, and output map_coordinates.json.
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
import json
|
|
import time
|
|
import requests
|
|
|
|
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
MAP_PAGES_DIR = os.path.join(CURRENT_DIR, "map_pages")
|
|
MAP_COORDS_OUTPUT = os.path.join(CURRENT_DIR, "map_coordinates.json")
|
|
|
|
os.makedirs(MAP_PAGES_DIR, exist_ok=True)
|
|
|
|
|
|
def download_maps():
|
|
print("Downloading RIT transport map pages live from fit25.com...")
|
|
headers = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
|
}
|
|
|
|
downloaded_count = 0
|
|
# Try downloading maps from 00 to 45
|
|
for i in range(46):
|
|
url = f"https://fit25.com/ritRouteMap/map{i:02d}.php?cb={int(time.time())}"
|
|
filepath = os.path.join(MAP_PAGES_DIR, f"map{i:02d}.html")
|
|
try:
|
|
print(f"Fetching Map {i:02d}: {url} ...")
|
|
res = requests.get(url, headers=headers, timeout=15)
|
|
if res.status_code == 200:
|
|
html = res.text
|
|
if "newpoints" in html or "GPolyline" in html:
|
|
with open(filepath, "w", encoding="utf-8") as f:
|
|
f.write(html)
|
|
print(f" -> Saved {len(html)} bytes to {os.path.basename(filepath)}")
|
|
downloaded_count += 1
|
|
else:
|
|
print(f" -> Skipped (no map data found in response)")
|
|
else:
|
|
print(f" -> Skipped (status code {res.status_code})")
|
|
|
|
# Respectful delay
|
|
time.sleep(0.5)
|
|
except Exception as e:
|
|
print(f" -> Error fetching map {i:02d}: {e}")
|
|
time.sleep(1.0)
|
|
|
|
print(f"Downloaded {downloaded_count} map page HTML files.")
|
|
|
|
|
|
def extract_stops(html_text: str) -> list:
|
|
stops = []
|
|
# Regex to capture newpoints entries, handling potential whitespace variation
|
|
pattern = re.compile(
|
|
r"new\s+Array\s*\(\s*"
|
|
r"([\d.]+)\s*,\s*([\d.]+)\s*,"
|
|
r"\s*[^,]+\s*,"
|
|
r"\s*[^,]+\s*,"
|
|
r"\s*'([^']*)'"
|
|
r"\s*\)",
|
|
re.IGNORECASE
|
|
)
|
|
for m in pattern.finditer(html_text):
|
|
try:
|
|
lat = float(m.group(1))
|
|
lng = float(m.group(2))
|
|
label = m.group(3).strip()
|
|
|
|
# Match formats like:
|
|
# - 'Route no:1 - Lift Gate'
|
|
# - 'Route no:9Vyasarpadi'
|
|
# - 'Route no: 9MKB Nagar'
|
|
# - 'Route no:13 - ICF'
|
|
# - 'Route no:14A - Kakallur'
|
|
label_match = re.search(r"Route\s+no\s*:\s*([a-zA-Z0-9]+)\s*-?\s*(.*)", label, re.IGNORECASE)
|
|
if label_match:
|
|
route_num = label_match.group(1).strip()
|
|
stop_name = label_match.group(2).strip()
|
|
# Clean up multiple spaces or hyphens in stop name
|
|
stop_name = re.sub(r'^-?\s*', '', stop_name)
|
|
stop_name = " ".join(stop_name.split())
|
|
if stop_name:
|
|
stops.append({"route_num": route_num, "name": stop_name, "lat": lat, "lng": lng})
|
|
except Exception as e:
|
|
print(f" Error parsing stop line: {m.group(0)} - {e}")
|
|
return stops
|
|
|
|
|
|
def extract_polylines(html_text: str) -> list:
|
|
polylines = []
|
|
poly_pattern = re.compile(r"GPolyline\s*\(\s*\[(.*?)\]", re.DOTALL | re.IGNORECASE)
|
|
coord_pattern = re.compile(r"GLatLng\s*\(\s*([\d.]+)\s*,\s*([\d.]+)\s*\)")
|
|
for poly_match in poly_pattern.finditer(html_text):
|
|
block = poly_match.group(1)
|
|
coords = []
|
|
for coord_match in coord_pattern.finditer(block):
|
|
coords.append([float(coord_match.group(1)), float(coord_match.group(2))])
|
|
if coords:
|
|
polylines.append(coords)
|
|
return polylines
|
|
|
|
|
|
def merge_polylines(polylines):
|
|
if not polylines:
|
|
return []
|
|
merged = list(polylines[0])
|
|
for seg in polylines[1:]:
|
|
if not seg:
|
|
continue
|
|
if merged and seg:
|
|
last = merged[-1]
|
|
first = seg[0]
|
|
# Connect if close, else just extend
|
|
if abs(last[0] - first[0]) < 0.001 and abs(last[1] - first[1]) < 0.001:
|
|
seg = seg[1:]
|
|
merged.extend(seg)
|
|
return merged
|
|
|
|
|
|
def parse_and_save_coordinates():
|
|
print("\nParsing downloaded map HTML files...")
|
|
all_routes = {}
|
|
|
|
html_files = sorted([f for f in os.listdir(MAP_PAGES_DIR) if f.endswith(".html")])
|
|
if not html_files:
|
|
print("Error: No downloaded HTML files found in map_pages directory!")
|
|
return
|
|
|
|
for filename in html_files:
|
|
filepath = os.path.join(MAP_PAGES_DIR, filename)
|
|
print(f"Parsing {filename} ...")
|
|
with open(filepath, "r", encoding="utf-8") as f:
|
|
content = f.read()
|
|
|
|
stops = extract_stops(content)
|
|
polylines = extract_polylines(content)
|
|
merged_poly = merge_polylines(polylines)
|
|
|
|
if not stops and not polylines:
|
|
print(f" -> No data parsed")
|
|
continue
|
|
|
|
route_nums = set()
|
|
for s in stops:
|
|
rn = s["route_num"]
|
|
# Clean route num (e.g. '01' -> '1', '14A' -> '14A')
|
|
rn_clean = re.sub(r'^0+', '', rn)
|
|
route_nums.add(rn_clean)
|
|
key = f"R{rn_clean}"
|
|
if key not in all_routes:
|
|
all_routes[key] = {"stops": [], "polyline": []}
|
|
all_routes[key]["stops"].append({
|
|
"name": s["name"],
|
|
"lat": s["lat"],
|
|
"lng": s["lng"]
|
|
})
|
|
|
|
if merged_poly:
|
|
# If stops are all for a single route, assign the polyline to it
|
|
if len(route_nums) == 1:
|
|
rn = list(route_nums)[0]
|
|
key = f"R{rn}"
|
|
existing = all_routes.get(key, {}).get("polyline", [])
|
|
if existing:
|
|
all_routes[key]["polyline"] = merge_polylines([existing, merged_poly])
|
|
else:
|
|
all_routes[key]["polyline"] = merged_poly
|
|
# Otherwise, try parsing route number from polyline variable comments or assign to all routes found
|
|
else:
|
|
# Fallback: assign polyline to all routes mentioned in this file
|
|
for rn in sorted(route_nums):
|
|
key = f"R{rn}"
|
|
if not all_routes[key].get("polyline"):
|
|
all_routes[key]["polyline"] = merged_poly
|
|
|
|
print(f" -> Found {len(stops)} stops, {len(polylines)} polyline segments. Associated routes: {sorted(route_nums)}")
|
|
|
|
# Deduplicate stops for each route
|
|
for key, data in all_routes.items():
|
|
seen = set()
|
|
unique = []
|
|
for s in data["stops"]:
|
|
ident = (s["name"].lower().strip(), round(s["lat"], 5), round(s["lng"], 5))
|
|
if ident not in seen:
|
|
seen.add(ident)
|
|
unique.append(s)
|
|
data["stops"] = unique
|
|
|
|
with open(MAP_COORDS_OUTPUT, "w", encoding="utf-8") as f:
|
|
json.dump(all_routes, f, indent=2, ensure_ascii=False)
|
|
|
|
total_stops = sum(len(d["stops"]) for d in all_routes.values())
|
|
total_wp = sum(len(d["polyline"]) for d in all_routes.values())
|
|
print(f"\nDone! Scraped {len(all_routes)} routes, {total_stops} stops, {total_wp} polyline waypoints.")
|
|
print(f"Output saved to: {MAP_COORDS_OUTPUT}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
download_maps()
|
|
parse_and_save_coordinates()
|