Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Add a page to debug memory usage

Daoud Clarke2acf4c41ce8e
diff --git a/mwmbl/templates/mwmbl/memory.html b/mwmbl/templates/mwmbl/memory.html
new file mode 100644
index 0000000..f623bcf
--- /dev/null
+++ b/mwmbl/templates/mwmbl/memory.html
@@ -0,0 +1,34 @@
+{% extends "base.html" %}
+{% load humanbytes %}
+
+{% block title %}<title>Mwmbl Debugging - Memory usage</title>{% endblock %}
+
+
+
+{% block content %}
+ <table>
+ <thead><tr><th>Memory type</th><th>Size</th></tr></thead>
+ <tr><td>rss</td><td>{{ memory_info.rss|human_bytes }}</td></tr>
+ <tr><td>vms</td><td>{{ memory_info.vms|human_bytes }}</td></tr>
+ <tr><td>shared</td><td>{{ memory_info.shared|human_bytes }}</td></tr>
+ <tr><td>text</td><td>{{ memory_info.text|human_bytes }}</td></tr>
+ <tr><td>lib</td><td>{{ memory_info.lib|human_bytes }}</td></tr>
+ <tr><td>data</td><td>{{ memory_info.data|human_bytes }}</td></tr>
+ <tr><td>dirty</td><td>{{ memory_info.dirty|human_bytes }}</td></tr>
+{# <tr><td>uss</td><td>{{ memory_info.uss|human_bytes }}</td></tr>#}
+{# <tr><td>pss</td><td>{{ memory_info.pss|human_bytes }}</td></tr>#}
+{# <tr><td>swap</td><td>{{ memory_info.swap|human_bytes }}</td></tr>#}
+ </table>
+ <br>
+ <table><thead><tr><th>Most common objects</th><th>Number</th></tr></thead>
+ {% for t, c in most_common_objects %}
+ <tr><td>{{ t }}</td><td>{{ c }}</td></tr>
+ {% endfor %}
+ </table>
+ <br>
+ <table><thead><tr><th>Growth objects</th><th>Number</th></tr></thead>
+ {% for t, c in growth %}
+ <tr><td>{{ t }}</td><td>{{ c }}</td></tr>
+ {% endfor %}
+ </table>
+{% endblock %}
\ No newline at end of file
diff --git a/mwmbl/templatetags/humanbytes.py b/mwmbl/templatetags/humanbytes.py
new file mode 100644
index 0000000..0044c9e
--- /dev/null
+++ b/mwmbl/templatetags/humanbytes.py
@@ -0,0 +1,57 @@
+"""
+https://stackoverflow.com/a/63839503/660902
+"""
+from django.template import Library
+
+
+class HumanBytes:
+ METRIC_LABELS = ["B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]
+ BINARY_LABELS = ["B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"]
+ PRECISION_OFFSETS = [0.5, 0.05, 0.005, 0.0005] # PREDEFINED FOR SPEED.
+ PRECISION_FORMATS = ["{}{:.0f} {}", "{}{:.1f} {}", "{}{:.2f} {}", "{}{:.3f} {}"] # PREDEFINED FOR SPEED.
+
+ @staticmethod
+ def format(num: int | float, metric: bool = False, precision: int = 1) -> str:
+ """
+ Human-readable formatting of bytes, using binary (powers of 1024)
+ or metric (powers of 1000) representation.
+ """
+
+ assert isinstance(num, (int, float)), "num must be an int or float"
+ assert isinstance(metric, bool), "metric must be a bool"
+ assert isinstance(precision, int) and 0 <= precision <= 3, "precision must be an int (range 0-3)"
+
+ unit_labels = HumanBytes.METRIC_LABELS if metric else HumanBytes.BINARY_LABELS
+ last_label = unit_labels[-1]
+ unit_step = 1000 if metric else 1024
+ unit_step_thresh = unit_step - HumanBytes.PRECISION_OFFSETS[precision]
+
+ is_negative = num < 0
+ if is_negative: # Faster than ternary assignment or always running abs().
+ num = abs(num)
+
+ for unit in unit_labels:
+ if num < unit_step_thresh:
+ # VERY IMPORTANT:
+ # Only accepts the CURRENT unit if we're BELOW the threshold where
+ # float rounding behavior would place us into the NEXT unit: F.ex.
+ # when rounding a float to 1 decimal, any number ">= 1023.95" will
+ # be rounded to "1024.0". Obviously we don't want ugly output such
+ # as "1024.0 KiB", since the proper term for that is "1.0 MiB".
+ break
+ if unit != last_label:
+ # We only shrink the number if we HAVEN'T reached the last unit.
+ # NOTE: These looped divisions accumulate floating point rounding
+ # errors, but each new division pushes the rounding errors further
+ # and further down in the decimals, so it doesn't matter at all.
+ num /= unit_step
+
+ return HumanBytes.PRECISION_FORMATS[precision].format("-" if is_negative else "", num, unit)
+
+
+register = Library()
+
+
[email protected](is_safe=True)
+def human_bytes(value: int) -> str:
+ return HumanBytes.format(value)
diff --git a/mwmbl/urls.py b/mwmbl/urls.py
index 0ac6ab2..e43cbde 100644
--- a/mwmbl/urls.py
+++ b/mwmbl/urls.py
@@ -22,7 +22,7 @@ from mwmbl.search_setup import queued_batches, ranker, batch_cache
from mwmbl.tinysearchengine import search
from mwmbl.views import home_fragment, add_url, index, approve, revert_current_curation, CurationDetailView, \
flag_curation, CurationFlagListView, flag_curation_update, domains_view, domain_view, CurationsView, submit_domain, \
- DomainSubmissionListView
+ DomainSubmissionListView, memory_view
urlpatterns = [
path('admin/', admin.site.urls),
@@ -50,4 +50,6 @@ urlpatterns = [
# New API
path("api/v1/search/", search.create_router(ranker, "1.0.0").urls),
path("api/v1/crawler/", crawler.create_router(batch_cache=batch_cache, queued_batches=queued_batches, version="1.0.0").urls),
+
+ path("debug/memory", memory_view, name="memory"),
]
diff --git a/mwmbl/views.py b/mwmbl/views.py
index 181c197..5d14be6 100644
--- a/mwmbl/views.py
+++ b/mwmbl/views.py
@@ -6,6 +6,8 @@ from typing import Optional
from urllib.parse import urlencode
import justext
+import objgraph
+import psutil
import requests
from django.conf import settings
from django.contrib.auth.decorators import login_required, permission_required
@@ -511,3 +513,15 @@ def domain_view(request, domain):
}
return render(request, "mwmbl/domain.html", context)
+
+
+def memory_view(request):
+ process = psutil.Process()
+ memory_info = process.memory_info()
+ most_common_objects = objgraph.most_common_types(limit=100)
+ growth = objgraph.growth(limit=100)
+ return render(request, "mwmbl/memory.html", {
+ "most_common_objects": most_common_objects,
+ "growth": growth,
+ "memory_info": memory_info,
+ })
diff --git a/poetry.lock b/poetry.lock
index dfec9cf..2e44154 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1302,6 +1302,20 @@ rsa = ["cryptography (>=3.0.0)"]
signals = ["blinker (>=1.4.0)"]
signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"]
+[[package]]
+name = "objgraph"
+version = "3.6.1"
+description = "Draws Python object reference graphs with graphviz"
+optional = false
+python-versions = ">=3.7"
+files = [
+ {file = "objgraph-3.6.1-py2.py3-none-any.whl", hash = "sha256:21c6bc62df0e7b77cc0a31d96feec04c965f09ec2e3d78b816b516a604f0defd"},
+ {file = "objgraph-3.6.1.tar.gz", hash = "sha256:fe96c74147bbcaae8665b396e5388bdcc3197deebba4e6381f05202ee5b453a7"},
+]
+
+[package.extras]
+ipython = ["graphviz"]
+
[[package]]
name = "packaging"
version = "24.0"
@@ -1467,6 +1481,35 @@ files = [
cymem = ">=2.0.2,<2.1.0"
murmurhash = ">=0.28.0,<1.1.0"
+[[package]]
+name = "psutil"
+version = "6.0.0"
+description = "Cross-platform lib for process and system monitoring in Python."
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7"
+files = [
+ {file = "psutil-6.0.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a021da3e881cd935e64a3d0a20983bda0bb4cf80e4f74fa9bfcb1bc5785360c6"},
+ {file = "psutil-6.0.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:1287c2b95f1c0a364d23bc6f2ea2365a8d4d9b726a3be7294296ff7ba97c17f0"},
+ {file = "psutil-6.0.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:a9a3dbfb4de4f18174528d87cc352d1f788b7496991cca33c6996f40c9e3c92c"},
+ {file = "psutil-6.0.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:6ec7588fb3ddaec7344a825afe298db83fe01bfaaab39155fa84cf1c0d6b13c3"},
+ {file = "psutil-6.0.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:1e7c870afcb7d91fdea2b37c24aeb08f98b6d67257a5cb0a8bc3ac68d0f1a68c"},
+ {file = "psutil-6.0.0-cp27-none-win32.whl", hash = "sha256:02b69001f44cc73c1c5279d02b30a817e339ceb258ad75997325e0e6169d8b35"},
+ {file = "psutil-6.0.0-cp27-none-win_amd64.whl", hash = "sha256:21f1fb635deccd510f69f485b87433460a603919b45e2a324ad65b0cc74f8fb1"},
+ {file = "psutil-6.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c588a7e9b1173b6e866756dde596fd4cad94f9399daf99ad8c3258b3cb2b47a0"},
+ {file = "psutil-6.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ed2440ada7ef7d0d608f20ad89a04ec47d2d3ab7190896cd62ca5fc4fe08bf0"},
+ {file = "psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fd9a97c8e94059b0ef54a7d4baf13b405011176c3b6ff257c247cae0d560ecd"},
+ {file = "psutil-6.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2e8d0054fc88153ca0544f5c4d554d42e33df2e009c4ff42284ac9ebdef4132"},
+ {file = "psutil-6.0.0-cp36-cp36m-win32.whl", hash = "sha256:fc8c9510cde0146432bbdb433322861ee8c3efbf8589865c8bf8d21cb30c4d14"},
+ {file = "psutil-6.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:34859b8d8f423b86e4385ff3665d3f4d94be3cdf48221fbe476e883514fdb71c"},
+ {file = "psutil-6.0.0-cp37-abi3-win32.whl", hash = "sha256:a495580d6bae27291324fe60cea0b5a7c23fa36a7cd35035a16d93bdcf076b9d"},
+ {file = "psutil-6.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:33ea5e1c975250a720b3a6609c490db40dae5d83a4eb315170c4fe0d8b1f34b3"},
+ {file = "psutil-6.0.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:ffe7fc9b6b36beadc8c322f84e1caff51e8703b88eee1da46d1e3a6ae11b4fd0"},
+ {file = "psutil-6.0.0.tar.gz", hash = "sha256:8faae4f310b6d969fa26ca0545338b21f73c6b15db7c4a8d934a5482faa818f2"},
+]
+
+[package.extras]
+test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"]
+
[[package]]
name = "psycopg2-binary"
version = "2.9.9"
@@ -2673,4 +2716,4 @@ indexer = ["Levenshtein", "beautifulsoup4", "idna", "langdetect", "pyarrow", "py
[metadata]
lock-version = "2.0"
python-versions = ">=3.10,<3.11"
-content-hash = "9cdf9693521aed6b8cc5a24eafa256bd0ac9ad1e0dcd01b6b80000ac85618c20"
+content-hash = "45fc86862c2dba353d6bd23111d063993023cef0f77474371333902d92b8e312"
diff --git a/pyproject.toml b/pyproject.toml
index 220263f..84dc93b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -44,6 +44,8 @@ pybloomfiltermmap3 = "^0.5.7"
lxml-html-clean = "^0.1.0"
lxml = "^5.2.0"
numpy = "1.23.2"
+objgraph = "^3.6.1"
+psutil = "^6.0.0"
[tool.poetry.extras]
indexer = [
"ujson",