An open source, non-profit web search engine
Add a page to debug memory usage
Daoud Clarke2acf4c41ce8e
diff --git a/mwmbl/templates/mwmbl/memory.html b/mwmbl/templates/mwmbl/memory.htmlnew file mode 100644index 0000000..f623bcf--- /dev/null+++ b/mwmbl/templates/mwmbl/memory.html@@ -0,0 +1,34 @@+{% extends "base.html" %}+{% load humanbytes %}++{% block title %}<title>Mwmbl Debugging - Memory usage</title>{% endblock %}++++{% block content %}+ <table>+ <thead><tr><th>Memory type</th><th>Size</th></tr></thead>+ <tr><td>rss</td><td>{{ memory_info.rss|human_bytes }}</td></tr>+ <tr><td>vms</td><td>{{ memory_info.vms|human_bytes }}</td></tr>+ <tr><td>shared</td><td>{{ memory_info.shared|human_bytes }}</td></tr>+ <tr><td>text</td><td>{{ memory_info.text|human_bytes }}</td></tr>+ <tr><td>lib</td><td>{{ memory_info.lib|human_bytes }}</td></tr>+ <tr><td>data</td><td>{{ memory_info.data|human_bytes }}</td></tr>+ <tr><td>dirty</td><td>{{ memory_info.dirty|human_bytes }}</td></tr>+{# <tr><td>uss</td><td>{{ memory_info.uss|human_bytes }}</td></tr>#}+{# <tr><td>pss</td><td>{{ memory_info.pss|human_bytes }}</td></tr>#}+{# <tr><td>swap</td><td>{{ memory_info.swap|human_bytes }}</td></tr>#}+ </table>+ <br>+ <table><thead><tr><th>Most common objects</th><th>Number</th></tr></thead>+ {% for t, c in most_common_objects %}+ <tr><td>{{ t }}</td><td>{{ c }}</td></tr>+ {% endfor %}+ </table>+ <br>+ <table><thead><tr><th>Growth objects</th><th>Number</th></tr></thead>+ {% for t, c in growth %}+ <tr><td>{{ t }}</td><td>{{ c }}</td></tr>+ {% endfor %}+ </table>+{% endblock %}\ No newline at end of filediff --git a/mwmbl/templatetags/humanbytes.py b/mwmbl/templatetags/humanbytes.pynew file mode 100644index 0000000..0044c9e--- /dev/null+++ b/mwmbl/templatetags/humanbytes.py@@ -0,0 +1,57 @@+"""+https://stackoverflow.com/a/63839503/660902+"""+from django.template import Library+++class HumanBytes:+ METRIC_LABELS = ["B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]+ BINARY_LABELS = ["B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"]+ PRECISION_OFFSETS = [0.5, 0.05, 0.005, 0.0005] # PREDEFINED FOR SPEED.+ PRECISION_FORMATS = ["{}{:.0f} {}", "{}{:.1f} {}", "{}{:.2f} {}", "{}{:.3f} {}"] # PREDEFINED FOR SPEED.++ @staticmethod+ def format(num: int | float, metric: bool = False, precision: int = 1) -> str:+ """+ Human-readable formatting of bytes, using binary (powers of 1024)+ or metric (powers of 1000) representation.+ """++ assert isinstance(num, (int, float)), "num must be an int or float"+ assert isinstance(metric, bool), "metric must be a bool"+ assert isinstance(precision, int) and 0 <= precision <= 3, "precision must be an int (range 0-3)"++ unit_labels = HumanBytes.METRIC_LABELS if metric else HumanBytes.BINARY_LABELS+ last_label = unit_labels[-1]+ unit_step = 1000 if metric else 1024+ unit_step_thresh = unit_step - HumanBytes.PRECISION_OFFSETS[precision]++ is_negative = num < 0+ if is_negative: # Faster than ternary assignment or always running abs().+ num = abs(num)++ for unit in unit_labels:+ if num < unit_step_thresh:+ # VERY IMPORTANT:+ # Only accepts the CURRENT unit if we're BELOW the threshold where+ # float rounding behavior would place us into the NEXT unit: F.ex.+ # when rounding a float to 1 decimal, any number ">= 1023.95" will+ # be rounded to "1024.0". Obviously we don't want ugly output such+ # as "1024.0 KiB", since the proper term for that is "1.0 MiB".+ break+ if unit != last_label:+ # We only shrink the number if we HAVEN'T reached the last unit.+ # NOTE: These looped divisions accumulate floating point rounding+ # errors, but each new division pushes the rounding errors further+ # and further down in the decimals, so it doesn't matter at all.+ num /= unit_step++ return HumanBytes.PRECISION_FORMATS[precision].format("-" if is_negative else "", num, unit)+++register = Library()++[email protected](is_safe=True)+def human_bytes(value: int) -> str:+ return HumanBytes.format(value)diff --git a/mwmbl/urls.py b/mwmbl/urls.pyindex 0ac6ab2..e43cbde 100644--- a/mwmbl/urls.py+++ b/mwmbl/urls.py@@ -22,7 +22,7 @@ from mwmbl.search_setup import queued_batches, ranker, batch_cachefrom mwmbl.tinysearchengine import searchfrom mwmbl.views import home_fragment, add_url, index, approve, revert_current_curation, CurationDetailView, \flag_curation, CurationFlagListView, flag_curation_update, domains_view, domain_view, CurationsView, submit_domain, \- DomainSubmissionListView+ DomainSubmissionListView, memory_viewurlpatterns = [path('admin/', admin.site.urls),@@ -50,4 +50,6 @@ urlpatterns = [# New APIpath("api/v1/search/", search.create_router(ranker, "1.0.0").urls),path("api/v1/crawler/", crawler.create_router(batch_cache=batch_cache, queued_batches=queued_batches, version="1.0.0").urls),++ path("debug/memory", memory_view, name="memory"),]diff --git a/mwmbl/views.py b/mwmbl/views.pyindex 181c197..5d14be6 100644--- a/mwmbl/views.py+++ b/mwmbl/views.py@@ -6,6 +6,8 @@ from typing import Optionalfrom urllib.parse import urlencodeimport justext+import objgraph+import psutilimport requestsfrom django.conf import settingsfrom django.contrib.auth.decorators import login_required, permission_required@@ -511,3 +513,15 @@ def domain_view(request, domain):}return render(request, "mwmbl/domain.html", context)+++def memory_view(request):+ process = psutil.Process()+ memory_info = process.memory_info()+ most_common_objects = objgraph.most_common_types(limit=100)+ growth = objgraph.growth(limit=100)+ return render(request, "mwmbl/memory.html", {+ "most_common_objects": most_common_objects,+ "growth": growth,+ "memory_info": memory_info,+ })diff --git a/poetry.lock b/poetry.lockindex dfec9cf..2e44154 100644--- a/poetry.lock+++ b/poetry.lock@@ -1302,6 +1302,20 @@ rsa = ["cryptography (>=3.0.0)"]signals = ["blinker (>=1.4.0)"]signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"]+[[package]]+name = "objgraph"+version = "3.6.1"+description = "Draws Python object reference graphs with graphviz"+optional = false+python-versions = ">=3.7"+files = [+ {file = "objgraph-3.6.1-py2.py3-none-any.whl", hash = "sha256:21c6bc62df0e7b77cc0a31d96feec04c965f09ec2e3d78b816b516a604f0defd"},+ {file = "objgraph-3.6.1.tar.gz", hash = "sha256:fe96c74147bbcaae8665b396e5388bdcc3197deebba4e6381f05202ee5b453a7"},+]++[package.extras]+ipython = ["graphviz"]+[[package]]name = "packaging"version = "24.0"@@ -1467,6 +1481,35 @@ files = [cymem = ">=2.0.2,<2.1.0"murmurhash = ">=0.28.0,<1.1.0"+[[package]]+name = "psutil"+version = "6.0.0"+description = "Cross-platform lib for process and system monitoring in Python."+optional = false+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7"+files = [+ {file = "psutil-6.0.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a021da3e881cd935e64a3d0a20983bda0bb4cf80e4f74fa9bfcb1bc5785360c6"},+ {file = "psutil-6.0.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:1287c2b95f1c0a364d23bc6f2ea2365a8d4d9b726a3be7294296ff7ba97c17f0"},+ {file = "psutil-6.0.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:a9a3dbfb4de4f18174528d87cc352d1f788b7496991cca33c6996f40c9e3c92c"},+ {file = "psutil-6.0.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:6ec7588fb3ddaec7344a825afe298db83fe01bfaaab39155fa84cf1c0d6b13c3"},+ {file = "psutil-6.0.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:1e7c870afcb7d91fdea2b37c24aeb08f98b6d67257a5cb0a8bc3ac68d0f1a68c"},+ {file = "psutil-6.0.0-cp27-none-win32.whl", hash = "sha256:02b69001f44cc73c1c5279d02b30a817e339ceb258ad75997325e0e6169d8b35"},+ {file = "psutil-6.0.0-cp27-none-win_amd64.whl", hash = "sha256:21f1fb635deccd510f69f485b87433460a603919b45e2a324ad65b0cc74f8fb1"},+ {file = "psutil-6.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c588a7e9b1173b6e866756dde596fd4cad94f9399daf99ad8c3258b3cb2b47a0"},+ {file = "psutil-6.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ed2440ada7ef7d0d608f20ad89a04ec47d2d3ab7190896cd62ca5fc4fe08bf0"},+ {file = "psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fd9a97c8e94059b0ef54a7d4baf13b405011176c3b6ff257c247cae0d560ecd"},+ {file = "psutil-6.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2e8d0054fc88153ca0544f5c4d554d42e33df2e009c4ff42284ac9ebdef4132"},+ {file = "psutil-6.0.0-cp36-cp36m-win32.whl", hash = "sha256:fc8c9510cde0146432bbdb433322861ee8c3efbf8589865c8bf8d21cb30c4d14"},+ {file = "psutil-6.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:34859b8d8f423b86e4385ff3665d3f4d94be3cdf48221fbe476e883514fdb71c"},+ {file = "psutil-6.0.0-cp37-abi3-win32.whl", hash = "sha256:a495580d6bae27291324fe60cea0b5a7c23fa36a7cd35035a16d93bdcf076b9d"},+ {file = "psutil-6.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:33ea5e1c975250a720b3a6609c490db40dae5d83a4eb315170c4fe0d8b1f34b3"},+ {file = "psutil-6.0.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:ffe7fc9b6b36beadc8c322f84e1caff51e8703b88eee1da46d1e3a6ae11b4fd0"},+ {file = "psutil-6.0.0.tar.gz", hash = "sha256:8faae4f310b6d969fa26ca0545338b21f73c6b15db7c4a8d934a5482faa818f2"},+]++[package.extras]+test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"]+[[package]]name = "psycopg2-binary"version = "2.9.9"@@ -2673,4 +2716,4 @@ indexer = ["Levenshtein", "beautifulsoup4", "idna", "langdetect", "pyarrow", "py[metadata]lock-version = "2.0"python-versions = ">=3.10,<3.11"-content-hash = "9cdf9693521aed6b8cc5a24eafa256bd0ac9ad1e0dcd01b6b80000ac85618c20"+content-hash = "45fc86862c2dba353d6bd23111d063993023cef0f77474371333902d92b8e312"diff --git a/pyproject.toml b/pyproject.tomlindex 220263f..84dc93b 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -44,6 +44,8 @@ pybloomfiltermmap3 = "^0.5.7"lxml-html-clean = "^0.1.0"lxml = "^5.2.0"numpy = "1.23.2"+objgraph = "^3.6.1"+psutil = "^6.0.0"[tool.poetry.extras]indexer = ["ujson",