ArabovMK's picture
fix assets.
8ec104b
Raw History Blame Contribute Delete
36.3 kB
# ============================================================================= # FULL STREAMLIT INTERFACE FOR TOPONYMIC RAG SYSTEM # ============================================================================= import streamlit as st import pandas as pd import numpy as np import json import pickle import os import sys import time import math from pathlib import Path import plotly.express as px import plotly.graph_objects as go from plotly.subplots import make_subplots import folium from streamlit_folium import st_folium import base64 from datetime import datetime import hashlib import glob from PIL import Image import geopandas as gpd # Page configuration (MUST BE THE FIRST STREAMLIT COMMAND) st.set_page_config( page_title="Toponymic RAG System", page_icon="๐Ÿ—บ๏ธ", layout="wide", initial_sidebar_state="expanded" ) # ============================================================================= # 1. LOAD ALL ARTIFACTS (ADAPTED) # ============================================================================= @st.cache_resource(show_spinner=False) def load_all_artifacts(): """Loads all artifacts from the folder toponym_rag_assets""" artifacts = {} artifacts_path = Path("toponym_rag_assets") if not artifacts_path.exists(): st.error(f"โŒ Folder {artifacts_path.absolute()} not found!") return None with st.spinner("๐Ÿ”„ Loading all artifacts..."): try: # 1. Load metadata (meta_index.json) meta_path = artifacts_path / "meta_index.json" if meta_path.exists(): with open(meta_path, "r", encoding="utf-8") as f: artifacts["meta_list"] = json.load(f) artifacts["df"] = pd.DataFrame(artifacts["meta_list"]) else: st.error("โŒ meta_index.json not found!") return None # 2. Load embeddings emb_path = artifacts_path / "embeddings.npy" if emb_path.exists(): artifacts["embeddings"] = np.load(emb_path) else: st.error("โŒ embeddings.npy not found!") return None # 3. Load FAISS index (lazily) artifacts["faiss_index"] = None # 4. Load embedding model (lazily) artifacts["embedder"] = None # 5. Load test queries test_path = artifacts_path / "test_queries.json" if test_path.exists(): with open(test_path, "r", encoding="utf-8") as f: artifacts["test_queries"] = json.load(f) else: artifacts["test_queries"] = [] # 6. Load metrics (final_results_paper1.csv) metrics_path = artifacts_path / "final_results_paper1.csv" if metrics_path.exists(): artifacts["metrics"] = pd.read_csv(metrics_path) else: artifacts["metrics"] = None # 7. Basic statistics (using English column names) artifacts["stats"] = { "total": len(artifacts["df"]), "types": artifacts["df"]["type"].nunique() if "type" in artifacts["df"].columns else 0, "objects": artifacts["df"]["geographic_object"].nunique() if "geographic_object" in artifacts["df"].columns else 0, "embedding_dim": artifacts["embeddings"].shape[1], "has_coords": artifacts["df"]["latitude"].notna().sum() } # 8. All CSV files (excluding meta_index.csv) artifacts["all_metrics"] = {} for csv_file in artifacts_path.glob("*.csv"): if csv_file.name != "meta_index.csv": df_name = csv_file.stem artifacts["all_metrics"][df_name] = pd.read_csv(csv_file) # 9. KD-Tree kdtree_path = artifacts_path / "kdtree.pkl" if kdtree_path.exists(): with open(kdtree_path, "rb") as f: artifacts["kdtree"] = pickle.load(f) # 10. GeoJSON geojson_path = artifacts_path / "toponyms_processed.geojson" if geojson_path.exists(): artifacts["gdf"] = gpd.read_file(geojson_path) # 11. All images (PNG) artifacts["images"] = {} for img_file in artifacts_path.glob("*.png"): artifacts["images"][img_file.stem] = str(img_file) # 12. All text files (like experiment_info.json as text) artifacts["texts"] = {} for txt_file in artifacts_path.glob("*.txt"): with open(txt_file, "r", encoding="utf-8") as f: artifacts["texts"][txt_file.stem] = f.read() # Also include experiment_info.json as readable text exp_info_path = artifacts_path / "experiment_info.json" if exp_info_path.exists(): with open(exp_info_path, "r", encoding="utf-8") as f: artifacts["texts"]["experiment_info"] = json.dumps(json.load(f), indent=2, ensure_ascii=False) # 13. README (not present, skip) artifacts["readme"] = None # 14. LaTeX table (not present, skip) artifacts["latex_table"] = None # 15. File statistics artifacts["file_stats"] = { "total_files": len(list(artifacts_path.glob("*"))), "csv_files": len(list(artifacts_path.glob("*.csv"))), "png_files": len(list(artifacts_path.glob("*.png"))), "json_files": len(list(artifacts_path.glob("*.json"))), "txt_files": len(list(artifacts_path.glob("*.txt"))), "other_files": len(list(artifacts_path.glob("*"))) - len(list(artifacts_path.glob("*.csv"))) - len(list(artifacts_path.glob("*.png"))) - len(list(artifacts_path.glob("*.json"))) - len(list(artifacts_path.glob("*.txt"))) } return artifacts except Exception as e: st.error(f"โŒ Loading error: {str(e)}") return None @st.cache_resource(show_spinner=False) def get_embedder(): """Loads the embedding model (cached separately)""" from sentence_transformers import SentenceTransformer return SentenceTransformer("intfloat/multilingual-e5-large") @st.cache_resource(show_spinner=False) def get_faiss_index(): """Loads the FAISS index (cached separately)""" import faiss return faiss.read_index(str(Path("toponym_rag_assets/faiss_index.index"))) # Load all artifacts artifacts = load_all_artifacts() if artifacts is None: st.stop() # ============================================================================= # 2. SEARCH FUNCTIONS (UPDATED COLUMN NAMES) # ============================================================================= def semantic_search(query, top_k=10): """Semantic search""" try: # Lazy loading of model and index if artifacts["embedder"] is None: artifacts["embedder"] = get_embedder() if artifacts["faiss_index"] is None: artifacts["faiss_index"] = get_faiss_index() # Encode query q_emb = artifacts["embedder"].encode( [query], normalize_embeddings=True, convert_to_numpy=True ).astype("float32") # Search scores, indices = artifacts["faiss_index"].search(q_emb, top_k) # Format results results = [] for score, idx in zip(scores[0], indices[0]): if idx < len(artifacts["meta_list"]): meta = artifacts["meta_list"][idx].copy() meta["score"] = float(score) results.append(meta) return results except Exception as e: st.error(f"Search error: {str(e)}") return [] def haversine_distance(lat1, lon1, lat2, lon2): """Haversine distance in km""" R = 6371 # Earth radius in km lat1, lon1, lat2, lon2 = map(np.radians, [lat1, lon1, lat2, lon2]) dlat = lat2 - lat1 dlon = lon2 - lon1 a = np.sin(dlat/2)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon/2)**2 c = 2 * np.arcsin(np.sqrt(a)) return R * c def spatial_filter(lat, lon, radius_km, df_filter=None): """Spatial filtering by radius""" if df_filter is None: df_filter = artifacts["df"] distances = df_filter.apply( lambda row: haversine_distance(lat, lon, row["latitude"], row["longitude"]), axis=1 ) mask = distances <= radius_km filtered_df = df_filter[mask].copy() filtered_df["distance_km"] = distances[mask] return filtered_df def hybrid_search(query, lat, lon, radius_km, top_k=10): """Hybrid search with geo-filtering""" # First spatial filtering filtered_df = spatial_filter(lat, lon, radius_km) if len(filtered_df) == 0: return [] # Then semantic search on filtered results try: if artifacts["embedder"] is None: artifacts["embedder"] = get_embedder() # Get embeddings for filtered results filtered_indices = filtered_df.index.tolist() filtered_embeddings = artifacts["embeddings"][filtered_indices] # Build temporary index import faiss temp_index = faiss.IndexFlatIP(artifacts["embeddings"].shape[1]) temp_index.add(filtered_embeddings) # Encode query q_emb = artifacts["embedder"].encode( [query], normalize_embeddings=True, convert_to_numpy=True ).astype("float32") # Search scores, indices = temp_index.search(q_emb, min(top_k, len(filtered_indices))) # Format results results = [] for score, idx in zip(scores[0], indices[0]): global_idx = filtered_indices[idx] meta = artifacts["df"].iloc[global_idx].to_dict() meta["score"] = float(score) meta["distance_km"] = haversine_distance(lat, lon, meta["latitude"], meta["longitude"]) results.append(meta) return results except Exception as e: st.error(f"Hybrid search error: {str(e)}") return [] # ============================================================================= # 3. VISUALIZATION FUNCTIONS (UPDATED COLUMN NAMES) # ============================================================================= def create_interactive_map(results, center_lat=None, center_lon=None): """Creates an interactive map with results""" if not results: return folium.Map(location=[55.5, 50.0], zoom_start=6) if center_lat is None or center_lon is None: center_lat = results[0]["latitude"] center_lon = results[0]["longitude"] # Create map m = folium.Map(location=[center_lat, center_lon], zoom_start=8) # Add clustering for results from folium.plugins import MarkerCluster marker_cluster = MarkerCluster().add_to(m) # Add result markers for r in results: popup_html = f""" <div style="font-family: Arial; width: 250px;"> <b>{r['name_russian']}</b><br> <i>{r.get('name_tatar', '')}</i><br><br> <b>Type:</b> {r['type']}<br> <b>Object:</b> {r['geographic_object']}<br> <b>Coordinates:</b> {r['latitude']:.4f}, {r['longitude']:.4f}<br> <b>Relevance:</b> {r.get('score', 0):.4f}<br> <b>Distance:</b> {r.get('distance_km', 0):.1f} km </div> """ # Color based on relevance score = r.get('score', 0) if score > 0.7: color = 'green' elif score > 0.5: color = 'orange' else: color = 'blue' folium.Marker( location=[r["latitude"], r["longitude"]], popup=folium.Popup(popup_html, max_width=300), icon=folium.Icon(color=color, icon='info-sign') ).add_to(marker_cluster) # Add search center if center_lat and center_lon: folium.Marker( [center_lat, center_lon], popup='Search center', icon=folium.Icon(color='red', icon='star') ).add_to(m) # Add radius circle (using radius from session state if available) radius = st.session_state.get("last_radius", 50) * 1000 # km to m folium.Circle( radius=radius, location=[center_lat, center_lon], color='red', fill=True, fillOpacity=0.1 ).add_to(m) return m def plot_score_distribution(results): """Score distribution plot""" if not results: return None df_plot = pd.DataFrame(results) fig = make_subplots( rows=1, cols=2, subplot_titles=("Relevance distribution", "Top results by score"), specs=[[{"type": "histogram"}, {"type": "bar"}]] ) fig.add_trace( go.Histogram(x=df_plot["score"], nbinsx=20, marker_color='blue'), row=1, col=1 ) top_by_score = df_plot.nlargest(10, "score")[["name_russian", "score"]] fig.add_trace( go.Bar( x=top_by_score["score"], y=top_by_score["name_russian"], orientation='h', marker_color='green' ), row=1, col=2 ) fig.update_layout(height=400, showlegend=False) return fig def plot_type_distribution(results): """Distribution by type""" if not results: return None df_plot = pd.DataFrame(results) type_counts = df_plot["type"].value_counts() fig = make_subplots( rows=1, cols=2, subplot_titles=("By type (pie)", "By type (bars)"), specs=[[{"type": "pie"}, {"type": "bar"}]] ) fig.add_trace( go.Pie( labels=type_counts.index, values=type_counts.values, hole=0.3 ), row=1, col=1 ) fig.add_trace( go.Bar( x=type_counts.index, y=type_counts.values, marker_color='orange' ), row=1, col=2 ) fig.update_layout(height=400) return fig def plot_distance_analysis(results, center_lat, center_lon): """Distance analysis""" if not results: return None df_plot = pd.DataFrame(results) if "distance_km" not in df_plot.columns: df_plot["distance_km"] = df_plot.apply( lambda r: haversine_distance(center_lat, center_lon, r["latitude"], r["longitude"]), axis=1 ) fig = make_subplots( rows=1, cols=2, subplot_titles=("Distance distribution", "Score vs Distance") ) fig.add_trace( go.Histogram(x=df_plot["distance_km"], nbinsx=20, marker_color='purple'), row=1, col=1 ) fig.add_trace( go.Scatter( x=df_plot["distance_km"], y=df_plot["score"], mode='markers', text=df_plot["name_russian"], marker=dict( size=10, color=df_plot["score"], colorscale='Viridis', showscale=True ) ), row=1, col=2 ) fig.update_layout(height=400) fig.update_xaxes(title_text="Distance (km)", row=1, col=1) fig.update_xaxes(title_text="Distance (km)", row=1, col=2) fig.update_yaxes(title_text="Score", row=1, col=2) return fig def plot_spatial_heatmap(results): """Spatial heatmap of distribution""" if not results: return None df_plot = pd.DataFrame(results) fig = px.density_map( df_plot, lat='latitude', lon='longitude', z='score', radius=20, center=dict(lat=df_plot["latitude"].mean(), lon=df_plot["longitude"].mean()), zoom=6, map_style="open-street-map", title="Relevance heatmap" ) fig.update_layout(height=500) return fig # ============================================================================= # 4. EXPORT FUNCTIONS (UNCHANGED) # ============================================================================= def get_table_download_link(df, filename="results.csv", text="๐Ÿ“ฅ Download CSV"): """Creates a download link for DataFrame""" csv = df.to_csv(index=False) b64 = base64.b64encode(csv.encode()).decode() href = f'<a href="data:file/csv;base64,{b64}" download="{filename}" style="text-decoration: none;">{text}</a>' return href def get_json_download_link(data, filename="results.json", text="๐Ÿ“ฅ Download JSON"): """Creates a download link for JSON""" json_str = json.dumps(data, ensure_ascii=False, indent=2) b64 = base64.b64encode(json_str.encode()).decode() href = f'<a href="data:file/json;base64,{b64}" download="{filename}" style="text-decoration: none;">{text}</a>' return href # ============================================================================= # 5. NEW FUNCTIONS FOR WORKING WITH ARTIFACTS # ============================================================================= def display_all_metrics(): """Displays all metrics from CSV files""" st.subheader("๐Ÿ“Š All metrics") if artifacts["all_metrics"]: tabs = st.tabs(list(artifacts["all_metrics"].keys())) for i, (name, df) in enumerate(artifacts["all_metrics"].items()): with tabs[i]: st.dataframe(df, width='stretch') # Basic visualization for numeric data numeric_cols = df.select_dtypes(include=[np.number]).columns if len(numeric_cols) >= 2: fig = px.line(df, x=numeric_cols[0], y=numeric_cols[1:]) st.plotly_chart(fig, width='stretch') else: st.info("No additional metrics") def display_all_images(): """Displays all images""" st.subheader("๐Ÿ–ผ๏ธ All visualizations") if artifacts["images"]: # Group images by categories (simple) for img_name, img_path in artifacts["images"].items(): st.write(f"**{img_name}**") st.image(img_path, width='stretch') else: st.info("No images") def display_all_texts(): """Displays all text files""" st.subheader("๐Ÿ“ Text reports") if artifacts["texts"]: selected = st.selectbox("Select report", list(artifacts["texts"].keys())) if selected: st.text_area( f"**{selected}**", artifacts["texts"][selected], height=400 ) # Download st.download_button( f"๐Ÿ“ฅ Download {selected}.txt", artifacts["texts"][selected], file_name=f"{selected}.txt" ) else: st.info("No text reports") # LaTeX table (not present) if artifacts["latex_table"]: with st.expander("๐Ÿ“œ LaTeX table for paper"): st.code(artifacts["latex_table"], language="latex") # README (not present) if artifacts["readme"]: with st.expander("๐Ÿ“– README"): st.markdown(artifacts["readme"]) def display_file_browser(): """File browser for all artifacts""" st.subheader("๐Ÿ“ Artifact file manager") artifacts_path = Path("toponym_rag_assets") files = sorted(artifacts_path.glob("*")) # Statistics by file type col1, col2, col3, col4 = st.columns(4) with col1: st.metric("Total files", artifacts["file_stats"]["total_files"]) with col2: st.metric("CSV", artifacts["file_stats"]["csv_files"]) with col3: st.metric("PNG", artifacts["file_stats"]["png_files"]) with col4: st.metric("JSON/TXT", artifacts["file_stats"]["json_files"] + artifacts["file_stats"]["txt_files"]) # Filter by extension extensions = list(set([f.suffix[1:] for f in files if f.suffix])) selected_ext = st.selectbox("Filter by type", ["All"] + sorted(extensions)) filtered_files = files if selected_ext != "All": filtered_files = [f for f in files if f.suffix == f".{selected_ext}"] # File table file_data = [] for f in filtered_files: size = f.stat().st_size modified = datetime.fromtimestamp(f.stat().st_mtime) # Format size if size < 1024: size_str = f"{size} B" elif size < 1024**2: size_str = f"{size/1024:.1f} KB" elif size < 1024**3: size_str = f"{size/1024**2:.1f} MB" else: size_str = f"{size/1024**3:.2f} GB" file_data.append({ "File": f.name, "Size": size_str, "Modified": modified.strftime("%Y-%m-%d %H:%M"), "Type": f.suffix[1:] if f.suffix else "" }) df_files = pd.DataFrame(file_data) st.dataframe(df_files, width='stretch') # Download selected file selected_file = st.selectbox("Select file to download", [f.name for f in filtered_files]) if selected_file: file_path = artifacts_path / selected_file with open(file_path, "rb") as f: st.download_button( f"๐Ÿ“ฅ Download {selected_file}", f, file_name=selected_file ) # ============================================================================= # 6. MAIN INTERFACE (EXTENDED WITH NEW TABS) # ============================================================================= # Sidebar with st.sidebar: st.image("https://img.icons8.com/color/96/000000/world-map--v1.png", width=100) st.title("๐Ÿ—บ๏ธ Toponymic RAG") st.markdown("---") # Dataset statistics (updated column names) st.subheader("๐Ÿ“Š Dataset statistics") col1, col2 = st.columns(2) with col1: st.metric("Total records", artifacts["stats"]["total"]) st.metric("Toponym types", artifacts["stats"]["types"]) with col2: st.metric("Geo-objects", artifacts["stats"]["objects"]) st.metric("With coordinates", artifacts["stats"]["has_coords"]) # Artifact statistics with st.expander("๐Ÿ“ฆ Artifact statistics"): st.metric("Total files", artifacts["file_stats"]["total_files"]) st.metric("CSV metrics", artifacts["file_stats"]["csv_files"]) st.metric("PNG visualizations", artifacts["file_stats"]["png_files"]) st.markdown("---") # Search settings st.subheader("โš™๏ธ Search settings") search_mode = st.radio( "Search mode", ["๐Ÿ” Semantic", "๐ŸŒ Hybrid (with geo)"], help="Hybrid search considers distance from the specified point" ) top_k = st.slider("Number of results", 5, 50, 10) if search_mode == "๐ŸŒ Hybrid (with geo)": st.markdown("---") st.subheader("๐Ÿ“ Geo-search parameters") # Preset centers preset_centers = { "Kazan": (55.7887, 49.1221), "Tyumen": (57.1535, 65.5343), "Moscow": (55.7558, 37.6173), "Custom": None } center_choice = st.selectbox("Search center", list(preset_centers.keys())) if center_choice == "Custom": col1, col2 = st.columns(2) with col1: lat = st.number_input("Latitude", value=55.5, format="%.4f") with col2: lon = st.number_input("Longitude", value=50.0, format="%.4f") else: lat, lon = preset_centers[center_choice] st.info(f"๐Ÿ“ {center_choice}: {lat:.4f}, {lon:.4f}") radius = st.slider("Search radius (km)", 10, 500, 100, 10) st.markdown("---") # Example queries st.subheader("๐Ÿ’ก Example queries") example_queries = [ "swamp", "river", "village", "lake", "mountain", "Kazan", "Tyumen", "ะั€ะณั‹ าฏะปำ™ั‚ ะนั‹ะปะณะฐัั‹" ] if st.button("๐ŸŽฒ Random example"): import random st.session_state["example_query"] = random.choice(example_queries) st.rerun() # Clear cache if st.button("๐Ÿ—‘๏ธ Clear cache"): st.cache_resource.clear() st.success("Cache cleared! Restart search.") # Main area st.title("๐Ÿ—บ๏ธ Intelligent toponym search") st.markdown("Semantic and geospatial search over Tatarstan toponym database") # Tabs tabs = st.tabs([ "๐Ÿ” Search", "๐Ÿ—บ๏ธ Map", "๐Ÿ“Š Analytics", "๐Ÿ“ˆ Metrics", "๐Ÿ“Š All metrics", "๐Ÿ–ผ๏ธ Visualizations", "๐Ÿ“ Reports", "๐Ÿ“ Files", "โ„น๏ธ Information" ]) # Tab 1: Search with tabs[0]: col1, col2 = st.columns([4, 1]) with col1: query = st.text_input( "Enter query:", value=st.session_state.get("example_query", ""), placeholder="For example: swamp, river, village...", key="query_input" ) with col2: search_button = st.button("๐Ÿ” Search", type="primary", width='stretch') if search_button and query: with st.spinner("๐Ÿ” Searching..."): start_time = time.time() if search_mode == "๐Ÿ” Semantic": results = semantic_search(query, top_k=top_k) search_type = "Semantic" else: results = hybrid_search(query, lat, lon, radius, top_k) search_type = f"Hybrid (center: {lat:.4f}, {lon:.4f}, radius: {radius} km)" search_time = time.time() - start_time if results: st.success(f"โœ… Found {len(results)} results in {search_time*1000:.1f} ms") # Save to session st.session_state["last_results"] = results st.session_state["last_query"] = query st.session_state["last_search_type"] = search_type if search_mode == "๐ŸŒ Hybrid (with geo)": st.session_state["last_center"] = (lat, lon) st.session_state["last_radius"] = radius # Results table (using English column names) df_results = pd.DataFrame(results) df_display = df_results[[ "name_russian", "type", "geographic_object", "latitude", "longitude", "score" ]].copy() if "distance_km" in df_results.columns: df_display["distance_km"] = df_results["distance_km"].round(1) df_display = df_display.round({ "latitude": 4, "longitude": 4, "score": 4 }) st.dataframe( df_display, width='stretch', hide_index=True, column_config={ "name_russian": "Name", "type": "Type", "geographic_object": "Object", "latitude": "Latitude", "longitude": "Longitude", "score": st.column_config.ProgressColumn("Relevance", format="%.4f", min_value=0, max_value=1), "distance_km": "Distance (km)" } ) # Export col1, col2, col3 = st.columns(3) with col1: st.markdown(get_table_download_link(df_results, f"results_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv"), unsafe_allow_html=True) with col2: st.markdown(get_json_download_link(results, f"results_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"), unsafe_allow_html=True) with col3: if st.button("๐Ÿ“‹ Copy to clipboard"): st.write("โœ… Copied (simulation)") else: st.warning("โŒ Nothing found. Try changing your query.") # Tab 2: Map with tabs[1]: if "last_results" in st.session_state and st.session_state["last_results"]: results = st.session_state["last_results"] st.subheader(f"๐Ÿ—บ๏ธ Results map for: '{st.session_state['last_query']}'") center_lat = st.session_state.get("last_center", (None, None))[0] center_lon = st.session_state.get("last_center", (None, None))[1] # Create map m = create_interactive_map(results, center_lat, center_lon) st_folium(m, width=1000, height=600) # Additional map settings with st.expander("โš™๏ธ Map settings"): col1, col2 = st.columns(2) with col1: if st.button("๐Ÿ”„ Refresh map"): st.rerun() with col2: map_style = st.selectbox("Map style", ["OpenStreetMap", "CartoDB positron", "CartoDB dark_matter"]) # Heatmap st.subheader("๐Ÿ”ฅ Distribution heatmap") heatmap_fig = plot_spatial_heatmap(results) if heatmap_fig: st.plotly_chart(heatmap_fig, width='stretch') else: st.info("๐Ÿ‘† First perform a search in the 'Search' tab") # Tab 3: Analytics with tabs[2]: if "last_results" in st.session_state and st.session_state["last_results"]: results = st.session_state["last_results"] st.subheader(f"๐Ÿ“Š Results analysis: '{st.session_state['last_query']}'") # Main metrics col1, col2, col3, col4 = st.columns(4) with col1: st.metric("Total results", len(results)) with col2: scores = [r["score"] for r in results] st.metric("Average score", f"{np.mean(scores):.4f}") with col3: st.metric("Max score", f"{np.max(scores):.4f}") with col4: st.metric("Min score", f"{np.min(scores):.4f}") st.markdown("---") # Charts col1, col2 = st.columns(2) with col1: score_fig = plot_score_distribution(results) if score_fig: st.plotly_chart(score_fig, width='stretch') with col2: type_fig = plot_type_distribution(results) if type_fig: st.plotly_chart(type_fig, width='stretch') # Distance analysis (if center exists) if "last_center" in st.session_state: center_lat, center_lon = st.session_state["last_center"] radius = st.session_state.get("last_radius", 100) dist_fig = plot_distance_analysis(results, center_lat, center_lon) if dist_fig: st.plotly_chart(dist_fig, width='stretch') # Distance statistics df_dist = pd.DataFrame(results) if "distance_km" in df_dist.columns: st.subheader("๐Ÿ“ Distance statistics") col1, col2, col3 = st.columns(3) with col1: st.metric("Average distance", f"{df_dist['distance_km'].mean():.1f} km") with col2: st.metric("Min distance", f"{df_dist['distance_km'].min():.1f} km") with col3: st.metric("Max distance", f"{df_dist['distance_km'].max():.1f} km") # Detailed table with st.expander("๐Ÿ“‹ Detailed table"): st.dataframe(pd.DataFrame(results), width='stretch') else: st.info("๐Ÿ‘† First perform a search in the 'Search' tab") # Tab 4: Metrics with tabs[3]: st.subheader("๐Ÿ“ˆ Search quality metrics") if artifacts["metrics"] is not None: st.dataframe(artifacts["metrics"], width='stretch') else: st.info("No metrics file found (final_results_paper1.csv)") # Our metrics from tests (hardcoded from your output) st.markdown("---") st.subheader("๐Ÿ“Š Testing results") col1, col2, col3 = st.columns(3) with col1: st.metric("Recall@1 (Semantic)", "77.4%") st.metric("Recall@1 (Hybrid)", "98.8%") with col2: st.metric("Recall@5 (Semantic)", "94.0%") st.metric("Recall@5 (Hybrid)", "100%") with col3: st.metric("MRR (Semantic)", "0.840") st.metric("MRR (Hybrid)", "0.994") st.markdown("---") # Spatial accuracy (example) st.subheader("๐Ÿ—บ๏ธ Spatial accuracy") spatial_data = { "Distance": ["0-10 km", "10-50 km", "50-100 km", "100-200 km", "200-500 km", "500-1000 km"], "Accuracy@1": [0.875, 0.500, 0.450, 0.511, 0.490, 0.833] } fig = px.bar( spatial_data, x="Distance", y="Accuracy@1", title="Accuracy dependency on distance from center", color="Accuracy@1", color_continuous_scale="Viridis" ) st.plotly_chart(fig, width='stretch') # Tab 5: All metrics with tabs[4]: display_all_metrics() # Tab 6: Visualizations with tabs[5]: display_all_images() # Tab 7: Reports with tabs[6]: display_all_texts() # Tab 8: Files with tabs[7]: display_file_browser() # Tab 9: Information with tabs[8]: st.subheader("โ„น๏ธ About the system") col1, col2 = st.columns(2) with col1: st.markdown(""" ### ๐Ÿ“‹ Description Intelligent toponym search system using: - **Semantic search** based on embeddings (multilingual-e5-large) - **Geospatial filtering** by radius - **Hybrid search** combining both approaches - **Interactive visualization** on map - **Analytics** and quality metrics ### ๐Ÿ“Š Characteristics """) st.json(artifacts["stats"]) with col2: st.markdown(""" ### ๐Ÿš€ Features - ๐Ÿ” **Semantic search** across the entire dataset - ๐ŸŒ **Geo-search** within a specified radius - ๐Ÿ—บ๏ธ **Interactive map** with clustering - ๐Ÿ“Š **Visualization** of distributions - ๐Ÿ“ฅ **Export** results to CSV/JSON - ๐Ÿ“ˆ **Metrics** and quality analysis ### ๐Ÿ“š Data - **Source:** TatarNLPWorld/tatarstan-toponyms - **Records:** 9,023 - **Types:** Microtoponyms, Toponyms - **Regions:** Tatarstan, Tyumen Oblast """) st.markdown("---") # Usage examples st.subheader("๐Ÿ’ก Usage examples") examples = [ ("๐Ÿ” Semantic search", "Find all objects related to water", "swamp lake river"), ("๐ŸŒ Hybrid search", "Find toponyms near Kazan", "Kazan, radius 50 km"), ("๐Ÿ“Š Analysis", "Explore distribution by type", "village vs city"), ("๐Ÿ—บ๏ธ Visualization", "View found results on map", "interactive map") ] for title, desc, example in examples: with st.expander(title): st.write(f"**{desc}**") st.code(example) st.markdown("---") # Version information st.subheader("๐Ÿ”ง Technical information") import sys import platform tech_info = { "Python": sys.version, "Streamlit": st.__version__, "Platform": platform.platform(), "Processor": platform.processor(), "Artifacts": str(Path("toponym_rag_assets").absolute()), "Total files": artifacts["file_stats"]["total_files"] } st.json(tech_info) # Footer st.markdown("---") st.markdown( """ <div style='text-align: center; color: gray; padding: 10px;'> ๐Ÿ—บ๏ธ Toponymic RAG System | Data: TatarNLPWorld | Model: multilingual-e5-large <br> <small>ยฉ 2026 | All rights reserved | Total artifacts: {total_files}</small> </div> """.format(total_files=artifacts["file_stats"]["total_files"]), unsafe_allow_html=True )
# =============================================================================
# FULL STREAMLIT INTERFACE FOR TOPONYMIC RAG SYSTEM
# =============================================================================
import streamlit as st
import pandas as pd
import numpy as np
import json
import pickle
import os
import sys
import time
import math
from pathlib import Path
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import folium
from streamlit_folium import st_folium
import base64
from datetime import datetime
import hashlib
import glob
from PIL import Image
import geopandas as gpd
# Page configuration (MUST BE THE FIRST STREAMLIT COMMAND)
st.set_page_config(
page_title="Toponymic RAG System",
page_icon="๐Ÿ—บ๏ธ",
layout="wide",
initial_sidebar_state="expanded"
)
# =============================================================================
# 1. LOAD ALL ARTIFACTS (ADAPTED)
# =============================================================================
@st.cache_resource(show_spinner=False)
def load_all_artifacts():
"""Loads all artifacts from the folder toponym_rag_assets"""
artifacts = {}
artifacts_path = Path("toponym_rag_assets")
if not artifacts_path.exists():
st.error(f"โŒ Folder {artifacts_path.absolute()} not found!")
return None
with st.spinner("๐Ÿ”„ Loading all artifacts..."):
try:
# 1. Load metadata (meta_index.json)
meta_path = artifacts_path / "meta_index.json"
if meta_path.exists():
with open(meta_path, "r", encoding="utf-8") as f:
artifacts["meta_list"] = json.load(f)
artifacts["df"] = pd.DataFrame(artifacts["meta_list"])
else:
st.error("โŒ meta_index.json not found!")
return None
# 2. Load embeddings
emb_path = artifacts_path / "embeddings.npy"
if emb_path.exists():
artifacts["embeddings"] = np.load(emb_path)
else:
st.error("โŒ embeddings.npy not found!")
return None
# 3. Load FAISS index (lazily)
artifacts["faiss_index"] = None
# 4. Load embedding model (lazily)
artifacts["embedder"] = None
# 5. Load test queries
test_path = artifacts_path / "test_queries.json"
if test_path.exists():
with open(test_path, "r", encoding="utf-8") as f:
artifacts["test_queries"] = json.load(f)
else:
artifacts["test_queries"] = []
# 6. Load metrics (final_results_paper1.csv)
metrics_path = artifacts_path / "final_results_paper1.csv"
if metrics_path.exists():
artifacts["metrics"] = pd.read_csv(metrics_path)
else:
artifacts["metrics"] = None
# 7. Basic statistics (using English column names)
artifacts["stats"] = {
"total": len(artifacts["df"]),
"types": artifacts["df"]["type"].nunique() if "type" in artifacts["df"].columns else 0,
"objects": artifacts["df"]["geographic_object"].nunique() if "geographic_object" in artifacts["df"].columns else 0,
"embedding_dim": artifacts["embeddings"].shape[1],
"has_coords": artifacts["df"]["latitude"].notna().sum()
}
# 8. All CSV files (excluding meta_index.csv)