RIT-nexus/chatbot-service/main.go

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"math"
	"net/http"
	"os"
	"regexp"
	"strings"
)

type FAQ struct {
	Keywords  []string `json:"keywords"`
	Question  string   `json:"question"`
	Questions []string `json:"questions"`
	Answer    string   `json:"answer"`
}

type IndexedDoc struct {
	FAQIndex int
	Question string
}

type ChatRequest struct {
	Message string `json:"message"`
}

type ChatResponse struct {
	Answer          string  `json:"answer"`
	MatchedQuestion string  `json:"matched_question,omitempty"`
	Confidence      float64 `json:"confidence"`
}

var (
	faqs         []FAQ
	indexedDocs  []IndexedDoc
	idf          map[string]float64
	docTerms     []map[string]float64 // term frequency for each document
	docLengths   []float64             // length of each document
	avgDocLength float64
	vocabSet     map[string]bool
)

var synonymMap = map[string]string{
	"timings":       "hours",
	"timing":        "hours",
	"schedule":      "hours",
	"time":          "hours",
	"fees":          "fee",
	"payment":       "fee",
	"pay":           "fee",
	"canteen":       "cafeteria",
	"mess":          "cafeteria",
	"food":          "cafeteria",
	"bus":           "transport",
	"buses":         "transport",
	"route":         "transport",
	"routes":        "transport",
	"admission":     "admissions",
	"enroll":        "admissions",
	"join":          "admissions",
	"phone":         "contact",
	"number":        "contact",
	"call":          "contact",
	"email":         "mail",
	"emailaddress":  "mail",
	"course":        "courses",
	"departments":   "courses",
	"branches":      "courses",
	"programs":      "courses",
	"hostels":       "hostel",
	"rooms":         "hostel",
	"stay":          "hostel",
	"accommodation": "hostel",
}

// Stopwords set to filter out noise
var stopwords = map[string]bool{
	"a": true, "about": true, "above": true, "after": true, "again": true, "against": true, "all": true, "am": true,
	"an": true, "and": true, "any": true, "are": true, "as": true, "at": true, "be": true, "because": true,
	"been": true, "before": true, "being": true, "below": true, "between": true, "both": true, "but": true, "by": true,
	"can": true, "could": true, "did": true, "do": true, "does": true, "doing": true, "down": true, "during": true,
	"each": true, "few": true, "for": true, "from": true, "further": true, "had": true, "has": true, "have": true,
	"having": true, "he": true, "her": true, "here": true, "hers": true, "herself": true, "him": true, "himself": true,
	"his": true, "how": true, "i": true, "if": true, "in": true, "into": true, "is": true, "it": true, "its": true,
	"itself": true, "me": true, "more": true, "most": true, "my": true, "myself": true, "no": true, "nor": true,
	"not": true, "of": true, "off": true, "on": true, "once": true, "only": true, "or": true, "other": true,
	"our": true, "ours": true, "ourselves": true, "out": true, "over": true, "own": true, "same": true, "she": true,
	"should": true, "so": true, "some": true, "such": true, "than": true, "that": true, "the": true, "their": true,
	"theirs": true, "them": true, "themselves": true, "then": true, "there": true, "these": true, "they": true,
	"this": true, "those": true, "through": true, "to": true, "too": true, "under": true, "until": true, "up": true,
	"very": true, "was": true, "we": true, "were": true, "what": true, "when": true, "where": true, "which": true,
	"while": true, "who": true, "whom": true, "why": true, "with": true, "you": true, "your": true, "yours": true,
	"yourself": true, "yourselves": true,
}

var cleanRegex = regexp.MustCompile(`[^a-z0-9\s]`)

func stem(word string) string {
	if strings.HasSuffix(word, "sses") {
		return word[:len(word)-2]
	}
	if strings.HasSuffix(word, "ies") {
		return word[:len(word)-3] + "y"
	}
	if strings.HasSuffix(word, "ss") {
		return word
	}
	if strings.HasSuffix(word, "s") && !strings.HasSuffix(word, "us") && !strings.HasSuffix(word, "as") && !strings.HasSuffix(word, "is") {
		return word[:len(word)-1]
	}
	if strings.HasSuffix(word, "eed") {
		return word[:len(word)-1]
	}
	if strings.HasSuffix(word, "ing") {
		w := word[:len(word)-3]
		if len(w) > 3 && w[len(w)-1] == w[len(w)-2] {
			c := w[len(w)-1]
			if c == 'n' || c == 't' || c == 'p' || c == 'd' || c == 'g' {
				w = w[:len(w)-1]
			}
		}
		return w
	}
	if strings.HasSuffix(word, "ed") {
		return word[:len(word)-2]
	}
	if strings.HasSuffix(word, "ly") {
		return word[:len(word)-2]
	}
	if strings.HasSuffix(word, "tional") {
		return word[:len(word)-6] + "tion"
	}
	return word
}

func getBigrams(tokens []string) []string {
	var bigrams []string
	for i := 0; i < len(tokens)-1; i++ {
		bigrams = append(bigrams, tokens[i]+"_"+tokens[i+1])
	}
	return bigrams
}

func soundex(word string) string {
	if len(word) == 0 {
		return ""
	}
	word = strings.ToLower(word)
	first := string(word[0])

	mappings := map[rune]rune{
		'b': '1', 'f': '1', 'p': '1', 'v': '1',
		'c': '2', 'g': '2', 'j': '2', 'k': '2', 'q': '2', 's': '2', 'x': '2', 'z': '2',
		'd': '3', 't': '3',
		'l': '4',
		'm': '5', 'n': '5',
		'r': '6',
	}

	var code []rune
	code = append(code, rune(first[0]))

	prevCode := '0'
	if c, exists := mappings[rune(word[0])]; exists {
		prevCode = c
	}

	for i := 1; i < len(word); i++ {
		r := rune(word[i])
		if r == 'a' || r == 'e' || r == 'i' || r == 'o' || r == 'u' || r == 'y' || r == 'h' || r == 'w' {
			continue
		}
		if c, exists := mappings[r]; exists {
			if c != prevCode {
				code = append(code, c)
				prevCode = c
			}
		} else {
			prevCode = '0'
		}
		if len(code) == 4 {
			break
		}
	}

	for len(code) < 4 {
		code = append(code, '0')
	}

	return string(code)
}

func tokenize(text string) []string {
	text = strings.ToLower(text)
	text = cleanRegex.ReplaceAllString(text, " ")
	words := strings.Fields(text)
	var tokens []string
	for _, word := range words {
		// Resolve synonyms
		if syn, exists := synonymMap[word]; exists {
			word = syn
		}
		// Apply stemming
		word = stem(word)
		if !stopwords[word] && len(word) > 1 {
			tokens = append(tokens, word)
		}
	}
	return tokens
}

func levenshtein(s, t string) int {
	sLen := len(s)
	tLen := len(t)
	if sLen == 0 {
		return tLen
	}
	if tLen == 0 {
		return sLen
	}

	d := make([][]int, sLen+1)
	for i := range d {
		d[i] = make([]int, tLen+1)
		d[i][0] = i
	}
	for j := range d[0] {
		d[0][j] = j
	}

	for i := 1; i <= sLen; i++ {
		for j := 1; j <= tLen; j++ {
			cost := 1
			if s[i-1] == t[j-1] {
				cost = 0
			}
			d[i][j] = minInt(d[i-1][j]+1, minInt(d[i][j-1]+1, d[i-1][j-1]+cost))
		}
	}
	return d[sLen][tLen]
}

func minInt(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func correctToken(token string) string {
	if vocabSet[token] {
		return token
	}
	if len(token) < 4 {
		return token
	}

	// Try Soundex first (phonetic correction)
	tokenSx := soundex(token)
	for vocabWord := range vocabSet {
		if soundex(vocabWord) == tokenSx {
			return vocabWord
		}
	}

	// Fallback to Levenshtein distance
	bestWord := token
	bestDist := 999

	for vocabWord := range vocabSet {
		diff := len(vocabWord) - len(token)
		if diff < 0 {
			diff = -diff
		}
		if diff > 2 {
			continue
		}

		dist := levenshtein(token, vocabWord)
		if dist < bestDist {
			bestDist = dist
			bestWord = vocabWord
		}
	}

	maxAllowedDist := 1
	if len(token) >= 6 {
		maxAllowedDist = 2
	}

	if bestDist <= maxAllowedDist {
		return bestWord
	}
	return token
}

func tokenizeQuery(query string) []string {
	tokens := tokenize(query)
	var corrected []string
	for _, token := range tokens {
		corrected = append(corrected, correctToken(token))
	}
	return corrected
}

func initTFIDF() { // keeping name initTFIDF for backwards compatibility
	// Build indexedDocs
	indexedDocs = nil
	for i, faq := range faqs {
		if faq.Question != "" {
			indexedDocs = append(indexedDocs, IndexedDoc{FAQIndex: i, Question: faq.Question})
		}
		for _, q := range faq.Questions {
			if q != "" {
				indexedDocs = append(indexedDocs, IndexedDoc{FAQIndex: i, Question: q})
			}
		}
	}

	allTokens := make([][]string, len(indexedDocs))
	var totalLength float64

	for i, doc := range indexedDocs {
		faq := faqs[doc.FAQIndex]
		docText := doc.Question + " " + strings.Join(faq.Keywords, " ")
		tokens := tokenize(docText)
		bigrams := getBigrams(tokens)
		tokens = append(tokens, bigrams...)
		allTokens[i] = tokens
		totalLength += float64(len(tokens))
	}

	avgDocLength = totalLength / float64(len(indexedDocs))

	docFrequencies := make(map[string]int)
	docTerms = make([]map[string]float64, len(indexedDocs))
	docLengths = make([]float64, len(indexedDocs))

	for i, tokens := range allTokens {
		docLengths[i] = float64(len(tokens))
		tfMap := make(map[string]float64)
		for _, token := range tokens {
			tfMap[token]++
		}
		docTerms[i] = tfMap

		uniqueTokens := make(map[string]bool)
		for _, token := range tokens {
			uniqueTokens[token] = true
		}
		for token := range uniqueTokens {
			docFrequencies[token]++
		}
	}

	// Populate vocabSet
	vocabSet = make(map[string]bool)
	for _, tokens := range allTokens {
		for _, token := range tokens {
			vocabSet[token] = true
		}
	}

	// Calculate BM25 IDF for each term
	idf = make(map[string]float64)
	numDocs := float64(len(indexedDocs))
	for token, df := range docFrequencies {
		val := (numDocs - float64(df) + 0.5) / (float64(df) + 0.5)
		if val < 0 {
			val = 0.0001
		}
		idf[token] = math.Log(val + 1.0)
	}

	log.Printf("BM25 Chatbot engine initialized successfully with %d Q&As (%d indexed question variants, avg doc length: %.2f).", len(faqs), len(indexedDocs), avgDocLength)
}

func getBestMatch(query string) (int, float64, string) {
	queryTokens := tokenizeQuery(query)
	if len(queryTokens) == 0 {
		return -1, 0.0, ""
	}

	qBigrams := getBigrams(queryTokens)
	queryTokens = append(queryTokens, qBigrams...)

	bestFAQIdx := -1
	bestScore := -1.0
	var bestMatchedQuestion string

	k1 := 1.2
	b := 0.75

	for i, doc := range indexedDocs {
		faq := faqs[doc.FAQIndex]
		var bm25Score float64

		tfMap := docTerms[i]
		docLen := docLengths[i]

		for _, token := range queryTokens {
			tf := tfMap[token]
			if tf > 0 {
				idfVal := idf[token]
				numerator := tf * (k1 + 1.0)
				denominator := tf + k1*(1.0-b+b*(docLen/avgDocLength))
				bm25Score += idfVal * (numerator / denominator)
			}
		}

		// Apply exact keyword boosting (only on original tokens)
		originalQueryLen := len(queryTokens) - len(qBigrams)
		keywordMatches := 0
		for _, kw := range faq.Keywords {
			for j := 0; j < originalQueryLen; j++ {
				qTok := queryTokens[j]
				if strings.ToLower(kw) == qTok {
					keywordMatches++
				}
			}
		}

		boost := float64(keywordMatches) * 0.15
		score := bm25Score + boost

		if score > bestScore {
			bestScore = score
			bestFAQIdx = doc.FAQIndex
			bestMatchedQuestion = doc.Question
		}
	}

	return bestFAQIdx, bestScore, bestMatchedQuestion
}

func handleChat(w http.ResponseWriter, r *http.Request) {
	// Add CORS headers
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS")
	w.Header().Set("Access-Control-Allow-Headers", "Content-Type")

	if r.Method == http.MethodOptions {
		w.WriteHeader(http.StatusOK)
		return
	}

	if r.Method != http.MethodPost {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}

	var req ChatRequest
	err := json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}

	bestIdx, score, matchedQuestion := getBestMatch(req.Message)
	threshold := 0.18

	var resp ChatResponse
	if bestIdx != -1 && score >= threshold {
		resp = ChatResponse{
			Answer:          faqs[bestIdx].Answer,
			MatchedQuestion: matchedQuestion,
			Confidence:      score,
		}
		log.Printf("[CHATBOT] Query: %q | MATCHED: %q (Score: %.4f)", req.Message, matchedQuestion, score)
	} else {
		resp = ChatResponse{
			Answer:     "I'm sorry, I couldn't find an answer to your question about RIT Chennai. Please try rephrasing your question or contact our administrative office at +91 8925977445 or mail@ritchennai.edu.in.",
			Confidence: score,
		}
		if bestIdx != -1 {
			log.Printf("[CHATBOT] Query: %q | FALLBACK (Close Match: %q, Score: %.4f < %.2f)", req.Message, matchedQuestion, score, threshold)
		} else {
			log.Printf("[CHATBOT] Query: %q | FALLBACK (No matches found)", req.Message)
		}
	}

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(resp)
}

func handleHealth(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	w.Write([]byte(`{"status":"UP"}`))
}

func main() {
	// Load FAQs
	file, err := os.ReadFile("qna.json")
	if err != nil {
		log.Fatalf("Failed to read qna.json: %v", err)
	}

	err = json.Unmarshal(file, &faqs)
	if err != nil {
		log.Fatalf("Failed to parse qna.json: %v", err)
	}

	initTFIDF()

	port := os.Getenv("PORT")
	if port == "" {
		port = "8081"
	}

	http.HandleFunc("/api/chat", handleChat)
	http.HandleFunc("/api/health", handleHealth)

	log.Printf("Chatbot service listening on port %s...", port)
	if err := http.ListenAndServe(fmt.Sprintf(":%s", port), nil); err != nil {
		log.Fatalf("Server failed to start: %v", err)
	}
}