Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

[Feat]: Add OpenTelemetry instrumentation for Monitoring LLM Applications using Vertex AI (#214)

patcher93e85e6119060
diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml
index 9135bba1..0163d898 100644
--- a/.github/workflows/python-tests.yml
+++ b/.github/workflows/python-tests.yml
@@ -35,6 +35,13 @@ jobs:
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
+
+ - name: Set up GCP Credentials
+ uses: google-github-actions/setup-gcloud@v0
+ with:
+ service_account_key: ${{ secrets.GCP_SA_KEY }}
+ project_id: ${{ secrets.GCP_PROJECT_ID }}
+ export_default_credentials: true
- name: Install dependencies
working-directory: sdk/python
diff --git a/assets/pricing.json b/assets/pricing.json
index 1d830df4..3927230d 100644
--- a/assets/pricing.json
+++ b/assets/pricing.json
@@ -21,7 +21,11 @@
"embed-multilingual-v2.0": 0.0001,
"mistral-embed": 0.0001,
"amazon.titan-embed-text-v1": 0.0001,
- "amazon.titan-embed-text-v2": 0.00002
+ "amazon.titan-embed-text-v2": 0.00002,
+ "textembedding-gecko": 0.0001,
+ "textembedding-gecko@001": 0.0001,
+ "textembedding-gecko@002": 0.0001,
+ "textembedding-gecko@003": 0.0001
},
"images": {
"dall-e-3": {
@@ -250,6 +254,58 @@
"ai21.j2-ultra-v1": {
"promptPrice": 0.0188,
"completionPrice": 0.0188
+ },
+ "gemini-1.0-pro-002": {
+ "promptPrice": 0.0005,
+ "completionPrice": 0.0015
+ },
+ "gemini-1.0-pro-001": {
+ "promptPrice": 0.0005,
+ "completionPrice": 0.0015
+ },
+ "gemini-1.5-pro-preview-0409": {
+ "promptPrice": 0.01,
+ "completionPrice": 0.03
+ },
+ "text-bison": {
+ "promptPrice": 0.001,
+ "completionPrice": 0.002
+ },
+ "text-bison@002": {
+ "promptPrice": 0.001,
+ "completionPrice": 0.002
+ },
+ "text-bison-32k": {
+ "promptPrice": 0.001,
+ "completionPrice": 0.002
+ },
+ "text-bison-32k@002": {
+ "promptPrice": 0.001,
+ "completionPrice": 0.002
+ },
+ "text-unicorn": {
+ "promptPrice": 0.01,
+ "completionPrice": 0.03
+ },
+ "text-unicorn@001": {
+ "promptPrice": 0.01,
+ "completionPrice": 0.03
+ },
+ "chat-bison": {
+ "promptPrice": 0.001,
+ "completionPrice": 0.002
+ },
+ "chat-bison@002": {
+ "promptPrice": 0.001,
+ "completionPrice": 0.002
+ },
+ "chat-bison-32k": {
+ "promptPrice": 0.001,
+ "completionPrice": 0.002
+ },
+ "chat-bison-32k@002": {
+ "promptPrice": 0.001,
+ "completionPrice": 0.002
}
}
}
diff --git a/docs/images/docs-vertexai-banner.gif b/docs/images/docs-vertexai-banner.gif
new file mode 100644
index 00000000..d09e475d
Binary files /dev/null and b/docs/images/docs-vertexai-banner.gif differ
diff --git a/docs/latest/integrations/bedrock.mdx b/docs/latest/integrations/bedrock.mdx
index 9ed69611..6b6d86c3 100644
--- a/docs/latest/integrations/bedrock.mdx
+++ b/docs/latest/integrations/bedrock.mdx
@@ -1,6 +1,6 @@
---
title: 'Monitor Amazon Bedrock using OpenTelemetry'
-sidebarTitle: 'Bedrock'
+sidebarTitle: 'Amazon Bedrock'
---
<Frame>
diff --git a/docs/latest/integrations/introduction.mdx b/docs/latest/integrations/introduction.mdx
index 2eab9cd0..65df6f04 100644
--- a/docs/latest/integrations/introduction.mdx
+++ b/docs/latest/integrations/introduction.mdx
@@ -132,6 +132,39 @@ Start integrating and monitoring your applications with OpenLIT effortlessly. Ch
iconType="duotone"
>
</Card>
+ <Card
+ title="Vertex AI"
+ href="/latest/integrations/vertexai"
+ icon={
+ <svg version="1.1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 434 434">
+ <path d="M0 0 C4.42268767 2.30122756 8.22547123 5.18210418 12.1171875 8.27734375 C13.61559167 9.4460593 15.11564491 10.61266312 16.6171875 11.77734375 C17.38160156 12.3703125 18.14601562 12.96328125 18.93359375 13.57421875 C22.60745808 16.38401008 26.35773256 19.08370728 30.1171875 21.77734375 C38.74446441 27.96812289 47.29809389 34.2542059 55.83789062 40.56494141 C58.2570446 42.35081157 60.67884769 44.13302657 63.1015625 45.9140625 C71.16052887 51.83979697 79.19688658 57.79261415 87.19091797 63.80566406 C94.67507326 69.42852311 102.21937817 74.96926209 109.76171875 80.51367188 C112.33088465 82.40514441 114.89633424 84.30156522 117.4609375 86.19921875 C118.31389404 86.82973145 119.16685059 87.46024414 120.0456543 88.10986328 C121.67289497 89.31291618 123.29940513 90.51695802 124.92504883 91.72216797 C127.02325869 93.27201581 129.13047745 94.80846505 131.2421875 96.33984375 C131.81590088 96.76652344 132.38961426 97.19320312 132.98071289 97.6328125 C136.12506105 99.88360701 137.02166269 100.35430439 141.0546875 100.08984375 C141.73273438 99.96351563 142.41078125 99.8371875 143.109375 99.70703125 C145.33642867 99.32361908 147.42109743 99.2227134 149.6796875 99.21484375 C150.33839844 99.21226562 150.99710938 99.2096875 151.67578125 99.20703125 C154.2421875 99.33984375 154.2421875 99.33984375 156.21972656 99.78442383 C159.67004799 100.46155128 162.18117931 100.82448582 165.44213867 99.33496094 C167.77358079 97.75597817 169.94115925 96.06446482 172.1171875 94.27734375 C173.79116109 92.97330907 175.46565493 91.669942 177.140625 90.3671875 C177.9872168 89.6954248 178.83380859 89.02366211 179.70605469 88.33154297 C183.64692399 85.23666313 187.69488509 82.29322389 191.7421875 79.33984375 C193.32611465 78.1739834 194.90944672 77.00731417 196.4921875 75.83984375 C205.9921875 68.83984375 205.9921875 68.83984375 208.36791992 67.08935547 C209.94887733 65.92440526 211.52976942 64.75936641 213.1105957 63.59423828 C217.10474578 60.65057395 221.1000718 57.70853029 225.09765625 54.76953125 C232.50238213 49.3234748 239.89643116 43.86526609 247.2421875 38.33984375 C255.37452654 32.22275986 263.56723485 26.189092 271.76611328 20.16162109 C274.3209941 18.28186161 276.87368684 16.3991655 279.42578125 14.515625 C280.27422607 13.88962402 281.1226709 13.26362305 281.99682617 12.61865234 C283.61052457 11.42791628 285.22398488 10.23685748 286.8371582 9.04541016 C288.96929826 7.47293647 291.10522601 5.90575596 293.2421875 4.33984375 C293.90025391 3.85273926 294.55832031 3.36563477 295.23632812 2.86376953 C301.01562442 -1.36177354 305.22673578 -2.30995795 312.2421875 -1.66015625 C317.51540903 0.16518966 320.03909836 2.94703578 323.2421875 7.33984375 C325.06112627 12.79666005 325.20615841 18.29140556 322.7421875 23.52734375 C316.74869948 30.27001777 309.60257994 35.31591696 302.265625 40.484375 C297.81698246 43.62543993 293.43438687 46.85362812 289.0546875 50.08984375 C288.25466309 50.68079834 287.45463867 51.27175293 286.63037109 51.88061523 C284.99039177 53.09238318 283.35058181 54.30438035 281.7109375 55.51660156 C279.26529748 57.32277637 276.81680113 59.12500017 274.3671875 60.92578125 C266.96835922 66.36580997 259.58129143 71.81942531 252.2421875 77.33984375 C244.092298 83.47012896 235.8819614 89.51694634 227.66552734 95.55761719 C223.68896233 98.4821512 219.71571389 101.41118332 215.7421875 104.33984375 C210.81686835 107.96937769 205.89121682 111.59845915 200.96484375 115.2265625 C200.10117187 115.86271484 199.2375 116.49886719 198.34765625 117.15429688 C196.69281195 118.37318757 195.03786922 119.59194463 193.3828125 120.81054688 C192.63644531 121.36033203 191.89007813 121.91011719 191.12109375 122.4765625 C190.13991577 123.19912231 190.13991577 123.19912231 189.13891602 123.9362793 C188.51299561 124.39945557 187.8870752 124.86263184 187.2421875 125.33984375 C186.51942627 125.82743164 185.79666504 126.31501953 185.05200195 126.81738281 C182.22428168 129.19613145 182.36937211 131.82518819 181.8671875 135.40234375 C180.36553955 144.28504562 177.44972603 151.12707454 170.2421875 156.77734375 C161.85177101 162.43488173 154.29855831 164.25285636 144.2421875 163.33984375 C135.08740174 161.36552869 128.02309614 156.74774485 122.64453125 149.0546875 C119.92875381 144.18529063 119.20479118 140.63583048 119.1171875 135.08984375 C118.51458052 130.28588342 117.17794114 127.94494672 113.56640625 124.890625 C109.47008906 121.77220614 105.19542881 118.92747597 100.89111328 116.10595703 C96.73075671 113.33213036 92.74582122 110.33326556 88.7421875 107.33984375 C87.15975161 106.17196011 85.57641522 105.00529563 83.9921875 103.83984375 C61.82552083 87.50651042 39.65885417 71.17317708 17.4921875 54.83984375 C16.31499146 53.9725061 16.31499146 53.9725061 15.11401367 53.08764648 C13.5410795 51.92854862 11.96832365 50.76920874 10.39575195 49.60961914 C6.27978995 46.57507415 2.16021241 43.54561139 -1.96484375 40.5234375 C-2.81949219 39.89630859 -3.67414063 39.26917969 -4.5546875 38.62304688 C-6.18244358 37.42876932 -7.8112796 36.23596169 -9.44140625 35.04492188 C-23.36333251 24.82176522 -23.36333251 24.82176522 -24.7578125 17.33984375 C-25.00028553 11.37904832 -23.84278315 6.84378574 -19.7578125 2.33984375 C-13.51453196 -2.10382156 -6.99303851 -3.48930808 0 0 Z M138.3671875 121.96484375 C135.73067883 126.15223987 134.95015779 129.41958566 135.2421875 134.33984375 C136.84156664 139.23206231 139.70030581 142.85707979 144.2421875 145.33984375 C148.80430557 146.76914122 152.73880933 146.98159054 157.0546875 144.77734375 C161.30933459 141.9047166 163.59448535 138.32613033 165.1796875 133.46484375 C165.26412909 129.24276438 164.28914554 126.02436822 162.2421875 122.33984375 C154.68009526 114.77775151 146.12342042 114.88306587 138.3671875 121.96484375 Z " fill="#659CF5" transform="translate(106.7578125,290.66015625)"/>
+ <path d="M0 0 C4.42268767 2.30122756 8.22547123 5.18210418 12.1171875 8.27734375 C13.61559167 9.4460593 15.11564491 10.61266312 16.6171875 11.77734375 C17.38160156 12.3703125 18.14601562 12.96328125 18.93359375 13.57421875 C22.60745808 16.38401008 26.35773256 19.08370728 30.1171875 21.77734375 C38.74446441 27.96812289 47.29809389 34.2542059 55.83789062 40.56494141 C58.2570446 42.35081157 60.67884769 44.13302657 63.1015625 45.9140625 C71.16052887 51.83979697 79.19688658 57.79261415 87.19091797 63.80566406 C94.6766976 69.42974348 102.22232501 74.97219026 109.76611328 80.51806641 C112.3209941 82.39782589 114.87368684 84.280522 117.42578125 86.1640625 C118.27422607 86.79006348 119.1226709 87.41606445 119.99682617 88.06103516 C121.61052457 89.25177122 123.22398488 90.44283002 124.8371582 91.63427734 C129.27795823 94.90940984 133.74364197 98.14444831 138.2421875 101.33984375 C136.07885215 103.55281439 133.91252738 105.32692352 131.3671875 107.08984375 C124.90407348 111.87777564 121.11386998 117.7199266 119.6171875 125.65234375 C119.4315625 126.98265625 119.4315625 126.98265625 119.2421875 128.33984375 C114.48103615 126.78296536 111.02409609 124.11413977 107.1171875 121.08984375 C105.68423092 119.99786128 104.25060937 118.90675101 102.81640625 117.81640625 C102.06794434 117.24567383 101.31948242 116.67494141 100.54833984 116.08691406 C96.70380152 113.17440969 92.81674409 110.32021508 88.9296875 107.46484375 C87.31900689 106.27995272 85.70832983 105.09505685 84.09765625 103.91015625 C83.29505371 103.32008789 82.49245117 102.73001953 81.66552734 102.12207031 C77.68896232 99.19753629 73.71572479 96.26848942 69.7421875 93.33984375 C68.15887054 92.17315487 66.57553719 91.00648822 64.9921875 89.83984375 C49.15885417 78.17317708 33.32552083 66.50651042 17.4921875 54.83984375 C16.31499146 53.9725061 16.31499146 53.9725061 15.11401367 53.08764648 C13.5410795 51.92854862 11.96832365 50.76920874 10.39575195 49.60961914 C6.27978995 46.57507415 2.16021241 43.54561139 -1.96484375 40.5234375 C-2.81949219 39.89630859 -3.67414063 39.26917969 -4.5546875 38.62304688 C-6.18244358 37.42876932 -7.8112796 36.23596169 -9.44140625 35.04492188 C-23.36333251 24.82176522 -23.36333251 24.82176522 -24.7578125 17.33984375 C-25.00028553 11.37904832 -23.84278315 6.84378574 -19.7578125 2.33984375 C-13.51453196 -2.10382156 -6.99303851 -3.48930808 0 0 Z " fill="#ADCAF9" transform="translate(106.7578125,290.66015625)"/>
+ <path d="M0 0 C8.11188729 5.74768977 11.79357255 13.00899052 13.78125 22.56640625 C15.14104151 31.45140757 12.86795872 38.50435265 8.0859375 46.015625 C3.02148108 52.57487692 -4.55746435 56.94348366 -12.6875 58.3125 C-22.90294326 59.28312623 -31.53957547 57.24929589 -39.5625 50.6875 C-45.20182852 45.29072324 -49.21874725 38.11195262 -49.85546875 30.234375 C-49.94349296 20.446083 -47.64557358 11.70586484 -40.86328125 4.328125 C-28.53048414 -7.0112641 -14.39126858 -8.84689375 0 0 Z M-29.5625 16.9375 C-32.19900867 21.12489612 -32.97952971 24.39224191 -32.6875 29.3125 C-31.08812086 34.20471856 -28.22938169 37.82973604 -23.6875 40.3125 C-19.12538193 41.74179747 -15.19087817 41.95424679 -10.875 39.75 C-6.62035291 36.87737285 -4.33520215 33.29878658 -2.75 28.4375 C-2.66555841 24.21542063 -3.64054196 20.99702447 -5.6875 17.3125 C-13.24959224 9.75040776 -21.80626708 9.85572212 -29.5625 16.9375 Z " fill="#4285F4" transform="translate(274.6875,395.6875)"/>
+ <path d="M0 0 C4.65247561 2.67452849 6.6783032 5.00040259 8.67578125 10.08984375 C9.13779472 13.99390638 9.14380171 17.87655655 9.14453125 21.8046875 C9.15595154 23.4983419 9.15595154 23.4983419 9.16760254 25.22621155 C9.17838609 27.60903721 9.18126377 29.99191006 9.17675781 32.37475586 C9.17578543 36.01170959 9.21743238 39.64655366 9.26171875 43.28320312 C9.26692604 45.6028602 9.26961394 47.92252458 9.26953125 50.2421875 C9.29412415 51.8659153 9.29412415 51.8659153 9.31921387 53.52244568 C9.25654669 60.39959169 8.35284094 63.96877589 3.67578125 69.08984375 C-1.27302376 72.6575869 -6.4244484 72.92198252 -12.32421875 72.08984375 C-17.65258719 70.30023993 -20.67602011 66.97882586 -23.32421875 62.08984375 C-24.21458418 59.21793059 -24.45873513 56.61528196 -24.48535156 53.6159668 C-24.49976288 52.29890999 -24.49976288 52.29890999 -24.51446533 50.95524597 C-24.51871521 50.01353043 -24.52296509 49.07181488 -24.52734375 48.1015625 C-24.53304382 47.12500198 -24.5387439 46.14844147 -24.5446167 45.14228821 C-24.55405526 43.07704752 -24.56059613 41.01179185 -24.56445312 38.9465332 C-24.57415087 35.79937232 -24.60518215 32.65286686 -24.63671875 29.50585938 C-24.6432501 27.49609761 -24.64850005 25.4863312 -24.65234375 23.4765625 C-24.66469055 22.54091965 -24.67703735 21.60527679 -24.6897583 20.64128113 C-24.66586412 13.57521788 -23.64227641 8.76452566 -18.88671875 3.27734375 C-13.32130984 -1.47361508 -6.96382456 -2.72701029 0 0 Z " fill="#4184F3" transform="translate(328.32421875,152.91015625)"/>
+ <path d="M0 0 C4.18632132 3.33197003 6.64719279 6.64423284 7.43554688 11.9465332 C7.59044269 14.90917733 7.6169447 17.84588628 7.59375 20.8125 C7.5975769 21.8957959 7.60140381 22.9790918 7.60534668 24.09521484 C7.6080599 26.37793918 7.60074024 28.66069492 7.58398438 30.94335938 C7.5625677 34.42649004 7.5838076 37.90755807 7.609375 41.390625 C7.60673162 43.61458786 7.60160644 45.83854944 7.59375 48.0625 C7.60184692 49.09810059 7.60994385 50.13370117 7.61828613 51.20068359 C7.53723605 57.48685559 6.81441797 61.33081708 3.0625 66.4375 C-2.2421229 70.55128919 -6.22444424 71.98775047 -12.9375 71.4375 C-18.20989347 69.62757388 -21.88270757 67.20297618 -24.9375 62.4375 C-25.89354916 59.56935253 -26.07954136 57.70788838 -26.11450195 54.72583008 C-26.12733719 53.77086136 -26.14017242 52.81589264 -26.15339661 51.83198547 C-26.16078354 50.80075058 -26.16817047 49.76951569 -26.17578125 48.70703125 C-26.18788887 47.1104493 -26.18788887 47.1104493 -26.20024109 45.48161316 C-26.21407822 43.22628405 -26.22480203 40.97093402 -26.23266602 38.71557617 C-26.24984655 35.28067839 -26.29373341 31.84675119 -26.33789062 28.41210938 C-26.34802068 26.21875706 -26.3565358 24.02539658 -26.36328125 21.83203125 C-26.38078934 20.81091751 -26.39829742 19.78980377 -26.41633606 18.73774719 C-26.39758016 12.51437089 -25.66766033 7.56577685 -21.9375 2.4375 C-15.72086544 -2.63730372 -7.33670654 -3.64468648 0 0 Z " fill="#ADCAF9" transform="translate(137.9375,59.5625)"/>
+ <path d="M0 0 C4.52434032 2.30151225 7.11469205 5.69709571 8.859375 10.3515625 C9.41393164 14.16173037 9.3970515 17.91681494 9.35546875 21.76171875 C9.35692398 22.84738632 9.35837921 23.93305389 9.35987854 25.05162048 C9.358224 27.3385589 9.34671284 29.62550896 9.32592773 31.91235352 C9.29693987 35.40624396 9.30533597 38.89861871 9.31835938 42.39257812 C9.31212234 44.61979901 9.30373168 46.84701513 9.29296875 49.07421875 C9.29590439 50.11498062 9.29884003 51.15574249 9.30186462 52.2280426 C9.22116658 58.45012114 8.70837143 63.15308918 4.39453125 67.99609375 C-0.703125 72.0390625 -0.703125 72.0390625 -4.140625 72.3515625 C-5.213125 72.4546875 -6.285625 72.5578125 -7.390625 72.6640625 C-12.03947256 72.27665854 -15.34217545 71.05176468 -19.140625 68.3515625 C-22.68810698 63.89446976 -24.25514464 60.35221067 -24.30175781 54.69995117 C-24.31136536 53.75854782 -24.3209729 52.81714447 -24.33087158 51.84721375 C-24.33512146 50.83157852 -24.33937134 49.8159433 -24.34375 48.76953125 C-24.34945007 47.72259598 -24.35515015 46.67566071 -24.36102295 45.59700012 C-24.37047266 43.38035086 -24.37700682 41.16368754 -24.38085938 38.94702148 C-24.39055716 35.5627376 -24.42157813 32.17906301 -24.453125 28.79492188 C-24.45965598 26.6406286 -24.46490608 24.48633099 -24.46875 22.33203125 C-24.4810968 21.32246872 -24.4934436 20.31290619 -24.50616455 19.27275085 C-24.48674025 13.11840745 -24.00186056 8.416652 -19.60546875 3.70703125 C-13.46897635 -0.98558058 -7.65413104 -2.31858714 0 0 Z " fill="#ADCAF9" transform="translate(200.140625,199.6484375)"/>
+ <path d="M0 0 C4.48505858 2.00225829 6.79522798 4.51771463 9.16186523 8.80929565 C10.86245662 13.25427646 10.56712788 17.92928356 10.53125 22.62890625 C10.5350769 23.73676987 10.53890381 24.84463348 10.54284668 25.98606873 C10.54556005 28.32006747 10.53823938 30.65409693 10.52148438 32.98803711 C10.5000477 36.55456424 10.52131775 40.1190816 10.546875 43.68554688 C10.54423202 45.95833776 10.53910767 48.23112741 10.53125 50.50390625 C10.53934692 51.5665419 10.54744385 52.62917755 10.55578613 53.72401428 C10.47898709 59.80288547 10.06580634 63.37946963 6 68 C1.37794027 71.97712117 -2.9950093 73.26199785 -9 73 C-14.22730596 71.43180821 -17.55575547 69.23150042 -21 65 C-23.40216725 59.58974648 -23.30633432 54.40576475 -23.30078125 48.59765625 C-23.30506638 47.60172958 -23.3093515 46.60580292 -23.31376648 45.57969666 C-23.31949393 43.47932697 -23.32002176 41.37893745 -23.31567383 39.27856445 C-23.31251395 36.07663944 -23.33599306 32.87563907 -23.36132812 29.67382812 C-23.36360138 27.62760533 -23.36430449 25.58138006 -23.36328125 23.53515625 C-23.372491 22.58391373 -23.38170074 21.6326712 -23.39118958 20.65260315 C-23.34798698 14.40289609 -22.7669463 9.14009682 -18.46484375 4.2578125 C-12.20051106 -0.38836776 -7.7578153 -0.7107946 0 0 Z " fill="#659CF5" transform="translate(263,247)"/>
+ <path d="M0 0 C4.51767 3.614136 6.72857425 7.18728014 8.1875 12.75 C8.4797071 16.06106215 8.49162214 19.35721623 8.48828125 22.6796875 C8.49256638 23.6288504 8.4968515 24.57801331 8.50126648 25.55593872 C8.50699287 27.55795496 8.50752254 29.55999201 8.50317383 31.56201172 C8.5000175 34.60811258 8.52348186 37.65323802 8.54882812 40.69921875 C8.55110181 42.65104045 8.55180429 44.60286474 8.55078125 46.5546875 C8.559991 47.45702118 8.56920074 48.35935486 8.57868958 49.28903198 C8.52707386 56.42334995 7.53834681 62.17526302 2.5625 67.625 C-2.234784 70.64551215 -6.50444862 72.22334719 -12.21484375 71.40625 C-17.82458392 69.83813627 -20.56114116 67.5254333 -23.8125 62.75 C-25.04738965 60.28022071 -24.95932543 58.6073838 -24.98950195 55.84472656 C-25.00233719 54.82316467 -25.01517242 53.80160278 -25.02839661 52.74908447 C-25.03578354 51.64079285 -25.04317047 50.53250122 -25.05078125 49.390625 C-25.058853 48.2534906 -25.06692474 47.1163562 -25.07524109 45.94476318 C-25.08909098 43.53382903 -25.09980907 41.12287519 -25.10766602 38.71191406 C-25.12486496 35.0289172 -25.1687429 31.34682097 -25.21289062 27.6640625 C-25.22301853 25.32292328 -25.23153433 22.98177641 -25.23828125 20.640625 C-25.25578934 19.5404303 -25.27329742 18.4402356 -25.29133606 17.30670166 C-25.27387296 11.1336025 -25.01121969 7.40756252 -20.8125 2.75 C-13.95035824 -2.43161725 -7.94510913 -3.53115962 0 0 Z " fill="#4184F3" transform="translate(393.8125,106.25)"/>
+ <path d="M0 0 C4.31586588 2.25134995 6.77837016 5.01377157 8.9296875 9.31640625 C10.41422212 14.67601286 10.08185236 19.13164545 7.9296875 24.31640625 C4.35518091 28.68524764 0.49762911 31.71290683 -5.171875 32.671875 C-11.31449165 33.06238499 -15.41092641 31.26318035 -20.0703125 27.31640625 C-23.77969239 23.1283967 -24.34398963 17.76506914 -24.0703125 12.31640625 C-22.52324772 6.96118201 -19.79101428 3.34249713 -15.0703125 0.31640625 C-10.21665609 -1.30147922 -4.91398121 -1.62254097 0 0 Z " fill="#4184F3" transform="translate(328.0703125,286.68359375)"/>
+ <path d="M0 0 C4.31586588 2.25134995 6.77837016 5.01377157 8.9296875 9.31640625 C10.41422212 14.67601286 10.08185236 19.13164545 7.9296875 24.31640625 C3.76271844 29.40936843 -0.38498343 32.01322394 -6.9453125 32.75390625 C-12.96351343 32.48873648 -17.45435627 30.05666681 -21.5078125 25.69140625 C-23.87114681 20.58660415 -25.01285381 15.55431286 -23.40625 10.03515625 C-21.4350971 5.74028652 -19.1039829 2.9020924 -15.0703125 0.31640625 C-10.21665609 -1.30147922 -4.91398121 -1.62254097 0 0 Z " fill="#4184F3" transform="translate(328.0703125,105.68359375)"/>
+ <path d="M0 0 C4.90624714 2.27790046 7.5167579 5.21745967 10 10 C11.45215332 16.00223371 11.27105214 20.64033686 8 26 C3.88846116 30.47032878 0.48594875 32.78932638 -5.6875 33.3125 C-10.47465932 33.23315482 -14.11306851 31.75791148 -18 29 C-21.79364192 24.91453947 -22.82040909 20.96424683 -23.1640625 15.53125 C-22.8056662 10.00170706 -20.35807803 6.26107246 -16.375 2.5625 C-11.16197343 -0.86712274 -6.12168656 -0.56890019 0 0 Z " fill="#4184F3" transform="translate(327,239)"/>
+ <path d="M0 0 C3.86921543 3.05563957 5.85012236 5.8200986 6.43359375 10.69140625 C6.56706288 21.18208019 6.56706288 21.18208019 2.09765625 25.88671875 C-2.52440348 29.86383992 -6.89735305 31.1487166 -12.90234375 30.88671875 C-18.95030859 29.0723293 -23.0742855 26.10758329 -26.21484375 20.57421875 C-27.50061265 15.54803122 -27.50444329 11.50288068 -25.71484375 6.69921875 C-22.92285844 2.36682775 -20.46243032 -0.4097128 -15.71484375 -2.55078125 C-10.16669872 -3.36936002 -4.92278663 -2.88089156 0 0 Z " fill="#659CF5" transform="translate(266.90234375,202.11328125)"/>
+ <path d="M0 0 C3.71195444 1.88825509 6.34152631 4.49509371 8.19921875 8.2265625 C9.97456446 13.94128477 9.79715505 19.4412698 7.171875 24.7890625 C3.55711694 28.79460521 0.3857491 31.93468494 -5.15625 32.58984375 C-11.08815004 32.78688239 -14.31555682 31.85210216 -19.140625 28.3515625 C-23.52869402 23.61706698 -24.45329178 19.90560293 -24.3515625 13.62890625 C-23.89694492 8.72072017 -21.70818625 5.65434734 -18.078125 2.5390625 C-12.66099067 -1.49497371 -6.53026694 -1.97814655 0 0 Z " fill="#ADCAF9" transform="translate(200.140625,286.6484375)"/>
+ <path d="M0 0 C4.1796989 2.8497947 7.11864324 5.98092971 8.75 10.875 C9.27166057 15.84360315 9.3224051 19.73018979 7.0625 24.25 C3.79981492 27.95358847 0.45771139 31.60511373 -4.640625 32.11328125 C-10.73116678 32.27136908 -14.5240193 32.13990941 -19.25 27.875 C-23.58456803 23.07183002 -24.56321307 19.46274667 -24.4609375 13.15234375 C-24.00960138 8.27958531 -21.8309493 5.18914976 -18.25 2.0625 C-12.63336126 -2.03296575 -6.4763665 -2.24181917 0 0 Z " fill="#ADCAF9" transform="translate(136.25,193.125)"/>
+ <path d="M0 0 C5.00230198 2.45567552 8.49851535 5.9970307 11 11 C11.55713132 16.95701945 11.52180186 21.95208399 8 27 C3.97967021 31.0422988 0.43712092 33.13083687 -5.3125 33.25 C-10.26183812 33.16978382 -13.82709449 32.15523961 -17.44140625 28.5703125 C-21.52947334 23.88825275 -22.30813823 20.27462865 -22.2109375 14.27734375 C-21.75631992 9.36915767 -19.56756125 6.30278484 -15.9375 3.1875 C-10.74602946 -0.6784887 -6.36230412 -0.64792913 0 0 Z " fill="#ADCAF9" transform="translate(134,145)"/>
+ <path d="M0 0 C3.71195444 1.88825509 6.34152631 4.49509371 8.19921875 8.2265625 C9.97456446 13.94128477 9.79715505 19.4412698 7.171875 24.7890625 C3.55711694 28.79460521 0.3857491 31.93468494 -5.15625 32.58984375 C-11.08815004 32.78688239 -14.31555682 31.85210216 -19.140625 28.3515625 C-23.52869402 23.61706698 -24.45329178 19.90560293 -24.3515625 13.62890625 C-23.89694492 8.72072017 -21.70818625 5.65434734 -18.078125 2.5390625 C-12.66099067 -1.49497371 -6.53026694 -1.97814655 0 0 Z " fill="#ADCAF9" transform="translate(200.140625,105.6484375)"/>
+ <path d="M0 0 C4.90331294 2.27653815 7.5537698 5.19814071 10 10 C11.22039567 14.58066321 11.24386084 18.10076316 10 22.6875 C7.05931501 27.55800951 3.45407953 31.18197349 -2 33 C-9.12228538 33.46703511 -13.10002703 32.37270516 -18.75 27.75 C-22.24168425 23.48238591 -23.29033545 19.37839443 -22.875 13.9375 C-21.9798458 8.61802044 -19.3660644 5.29776118 -15.5 1.625 C-10.33499991 -0.77303575 -5.5997712 -0.75381535 0 0 Z " fill="#4184F3" transform="translate(391,58)"/>
+ <path d="M0 0 C4.46207218 2.57808615 7.16942068 5.02296667 8.671875 10.09375 C9.29132125 15.27671165 9.49032095 20.26935811 7.109375 25.03125 C3.67958115 29.34047817 0.63608987 31.63148498 -4.7578125 32.47265625 C-10.80109932 32.75737037 -14.51292409 31.75330269 -19.328125 28.09375 C-23.44191419 22.7891271 -24.87837547 18.80680576 -24.328125 12.09375 C-22.51819888 6.82135653 -20.09360118 3.14854243 -15.328125 0.09375 C-10.29313184 -1.58458105 -5.01588445 -1.95249193 0 0 Z " fill="#659CF5" transform="translate(264.328125,334.90625)"/>
+ <path d="M0 0 C4.52966583 2.47551505 7.16586374 5.37259122 8.8125 10.3125 C9.602593 16.66710515 9.09746175 21.4363026 5.28125 26.68359375 C2.3032312 29.98634332 -0.38621168 31.09721743 -4.76953125 31.62109375 C-11.13452321 31.90843265 -15.16935827 31.30653117 -20.1875 27.3125 C-23.90949833 22.54070727 -24.84545199 18.77691373 -24.5703125 12.7421875 C-23.73144217 7.41792888 -21.41611323 4.2653021 -17.125 0.9375 C-11.30410201 -2.28257123 -6.10135312 -2.82301413 0 0 Z " fill="#ADCAF9" transform="translate(136.1875,240.6875)"/>
+ <path d="M0 0 C4.47659907 3.45337643 7.0079023 6.0039838 8.1875 11.6875 C8.28488123 17.71431612 7.20375893 22.01995583 3.1875 26.6875 C-1.09458067 30.28834056 -5.20448828 31.94001835 -10.8125 31.6875 C-15.25987152 30.83255243 -18.64339467 29.26725813 -21.6875 25.875 C-25.12187053 20.7234442 -26.35693226 16.71948278 -25.34765625 10.640625 C-23.73037332 4.73813254 -19.9390678 1.63254959 -14.8125 -1.3125 C-9.80062777 -2.98312408 -4.75771859 -2.22026868 0 0 Z " fill="#4184F3" transform="translate(393.8125,241.3125)"/>
+ <path d="M0 0 C4.90331294 2.27653815 7.5537698 5.19814071 10 10 C11.22039567 14.58066321 11.24386084 18.10076316 10 22.6875 C7.08842448 27.50979695 4.13378031 31.09960405 -1.43359375 32.69921875 C-7.04316564 33.49326774 -11.02235762 32.92802493 -16 30 C-20.40203789 26.47836969 -22.05822226 23.55648868 -23 18 C-23.08827844 12.30604081 -21.72409035 8.42864798 -18 4 C-12.72867143 -1.00776214 -6.89190982 -0.92775709 0 0 Z " fill="#4184F3" transform="translate(391,192)"/>
+ <path d="M0 0 C3.920909 3.37380541 6.35769549 5.86252365 7.0703125 11.15625 C7.24846164 15.95440023 7.10097955 19.42192866 5.0625 23.8125 C0.48884327 28.6268755 -2.6212462 30.56406236 -9.3125 30.8125 C-14.43071229 30.62880179 -17.45307744 29.88246587 -21.3125 26.3125 C-25.31513619 21.87714639 -26.69013255 18.55466389 -26.58984375 12.50390625 C-25.90071822 7.058844 -23.51837449 3.29227302 -19.3125 -0.125 C-12.78920657 -3.52254866 -6.55981716 -3.34179365 0 0 Z " fill="#659CF5" transform="translate(266.3125,154.6875)"/>
+ <path d="M0 0 C4.52966583 2.47551505 7.16586374 5.37259122 8.8125 10.3125 C9.54377952 17.16974405 9.01833127 21.77851149 4.8125 27.3125 C0.19479082 31.81476645 -4.15079021 31.75398869 -10.28515625 31.74609375 C-16.5070414 30.81657928 -19.60500552 27.08915931 -23.1875 22.3125 C-24.98660157 16.91519529 -24.88218904 12.21201716 -22.5625 7.0625 C-17.92360787 -1.34033308 -8.56454307 -3.96269903 0 0 Z " fill="#ADCAF9" transform="translate(200.1875,153.6875)"/>
+ </svg>
+ }
+ iconType="duotone"
+ >
+ </Card>
</CardGroup>
### VectorDBs
diff --git a/docs/latest/integrations/vertexai.mdx b/docs/latest/integrations/vertexai.mdx
new file mode 100644
index 00000000..596b5554
--- /dev/null
+++ b/docs/latest/integrations/vertexai.mdx
@@ -0,0 +1,52 @@
+---
+title: 'Monitor Vertex AI using OpenTelemetry'
+sidebarTitle: 'Vertex AI'
+---
+
+<Frame>
+ <img src="/images/docs-vertexai-banner.gif" />
+</Frame>
+
+OpenLIT SDKs provide OpenTelemetry Auto-Instrumentation for monitoring LLM Application using Vertex AI which includes performance, token and cost usage and user interactions.
+
+The integration is compatible with
+- google-cloud-aiplatform `>= 1.38.1`
+
+## Get Started
+
+<Steps>
+ <Step title="Install the OpenLIT Python SDK using pip">
+ ```shell
+ pip install openlit
+ ```
+ </Step>
+ <Step title="Add the following two lines to your application code:">
+ Integrating the OpenLIT into LLM applications is straightforward. Start monitoring for your LLM Application with just **one line of code**:
+
+ ```python
+ import openlit
+
+ openlit.init(otlp_endpoint="otlp_backend_endpoint", otlp_headers="otlp_backend_token")
+ ```
+
+ Alternatively, you can configure the OTLP endpoint by setting the `OTEL_EXPORTER_OTLP_ENDPOINT` and the OTLP headers by setting the Appli`OTEL_EXPORTER_OTLP_HEADERS` environmentcation variable as recommended in the OpenTelemetry documentation.
+
+ If your aim is primarily to test or develop without sending telemetry data to an external backend, you can initialize OpenLIT without any parameters. This approach directs all telemetry output to your console, enabling quick diagnostics and local debugging:
+
+ ```python
+ import openlit
+
+ openlit.init()
+ ```
+
+ Refer to the OpenLIT [Python SDK repository](https://github.com/openlit/openlit/tree/main/sdk/python) for more advanced configurations and use cases.
+ </Step>
+</Steps>
+
+---
+
+<CardGroup cols={2}>
+<Card title="Connections" href="/latest/connections/intro" icon='link'>
+Connect to your existing Observablity Platform
+</Card>
+</CardGroup>
\ No newline at end of file
diff --git a/docs/mint.json b/docs/mint.json
index 281adab4..f0263da0 100644
--- a/docs/mint.json
+++ b/docs/mint.json
@@ -64,7 +64,9 @@
"latest/integrations/cohere",
"latest/integrations/mistral",
"latest/integrations/azure-openai",
- "latest/integrations/huggingface"
+ "latest/integrations/huggingface",
+ "latest/integrations/bedrock",
+ "latest/integrations/vertexai"
]
},
{
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml
index bb7f4f46..4835d2cd 100644
--- a/sdk/python/pyproject.toml
+++ b/sdk/python/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "openlit"
-version = "1.1.3"
+version = "1.2.0"
description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications, facilitating the integration of observability into your GenAI-driven projects"
authors = ["OpenLIT"]
repository = "https://github.com/openlit/openlit/tree/main/openlit/python"
diff --git a/sdk/python/src/openlit/__init__.py b/sdk/python/src/openlit/__init__.py
index 1e0dd816..339e627a 100644
--- a/sdk/python/src/openlit/__init__.py
+++ b/sdk/python/src/openlit/__init__.py
@@ -18,6 +18,7 @@ from openlit.instrumentation.anthropic import AnthropicInstrumentor
from openlit.instrumentation.cohere import CohereInstrumentor
from openlit.instrumentation.mistral import MistralInstrumentor
from openlit.instrumentation.bedrock import BedrockInstrumentor
+from openlit.instrumentation.vertexai import VertexAIInstrumentor
from openlit.instrumentation.langchain import LangChainInstrumentor
from openlit.instrumentation.chroma import ChromaInstrumentor
from openlit.instrumentation.pinecone import PineconeInstrumentor
@@ -139,7 +140,6 @@ def init(environment="default", application_name="default", tracer=None, otlp_en
disable_metrics (bool): Flag to disable metrics (Optional)
"""
disabled_instrumentors = disabled_instrumentors if disabled_instrumentors else []
-
# Check for invalid instrumentor names
module_name_map = {
@@ -148,6 +148,7 @@ def init(environment="default", application_name="default", tracer=None, otlp_en
"cohere": "cohere",
"mistral": "mistralai",
"bedrock": "boto3",
+ "vertexai": "vertexai",
"langchain": "langchain",
"chroma": "chromadb",
"pinecone": "pinecone",
@@ -195,6 +196,7 @@ def init(environment="default", application_name="default", tracer=None, otlp_en
"cohere": CohereInstrumentor(),
"mistral": MistralInstrumentor(),
"bedrock": BedrockInstrumentor(),
+ "vertexai": VertexAIInstrumentor(),
"langchain": LangChainInstrumentor(),
"chroma": ChromaInstrumentor(),
"pinecone": PineconeInstrumentor(),
diff --git a/sdk/python/src/openlit/instrumentation/vertexai/__init__.py b/sdk/python/src/openlit/instrumentation/vertexai/__init__.py
new file mode 100644
index 00000000..f89a4d0b
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/vertexai/__init__.py
@@ -0,0 +1,147 @@
+# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code
+"""Initializer of Auto Instrumentation of VertexAI Functions"""
+
+from typing import Collection
+import importlib.metadata
+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
+from wrapt import wrap_function_wrapper
+
+from openlit.instrumentation.vertexai.vertexai import (
+ generate_content, predict, predict_streaming,
+ send_message, start_chat, start_chat_streaming,
+ embeddings
+)
+from openlit.instrumentation.vertexai.async_vertexai import (
+ generate_content_async, predict_async,
+ predict_streaming_async,
+ send_message_async,
+ start_chat_async, start_chat_streaming_async,
+ embeddings_async
+)
+
+
+_instruments = ("google-cloud-aiplatform >= 1.38.1",)
+
+class VertexAIInstrumentor(BaseInstrumentor):
+ """
+ An instrumentor for VertexAI's client library.
+ """
+
+ def instrumentation_dependencies(self) -> Collection[str]:
+ return _instruments
+
+ def _instrument(self, **kwargs):
+ application_name = kwargs.get("application_name", "default")
+ environment = kwargs.get("environment", "default")
+ tracer = kwargs.get("tracer")
+ metrics = kwargs.get("metrics_dict")
+ pricing_info = kwargs.get("pricing_info", {})
+ trace_content = kwargs.get("trace_content", False)
+ disable_metrics = kwargs.get("disable_metrics")
+ version = importlib.metadata.version("google-cloud-aiplatform")
+
+ #sync
+ wrap_function_wrapper(
+ "vertexai.generative_models",
+ "GenerativeModel.generate_content",
+ generate_content("vertexai.generate_content", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "vertexai.generative_models",
+ "ChatSession.send_message",
+ send_message("vertexai.send_message", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "vertexai.language_models",
+ "TextGenerationModel.predict",
+ predict("vertexai.predict", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "vertexai.language_models",
+ "TextGenerationModel.predict_streaming",
+ predict_streaming("vertexai.predict", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "vertexai.language_models",
+ "ChatSession.send_message",
+ start_chat("vertexai.send_message", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "vertexai.language_models",
+ "ChatSession.send_message_streaming",
+ start_chat_streaming("vertexai.send_message", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "vertexai.language_models",
+ "TextEmbeddingModel.get_embeddings",
+ embeddings("vertexai.get_embeddings", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ #async
+ wrap_function_wrapper(
+ "vertexai.generative_models",
+ "GenerativeModel.generate_content_async",
+ generate_content_async("vertexai.generate_content", version, environment,
+ application_name, tracer, pricing_info, trace_content,
+ metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "vertexai.generative_models",
+ "ChatSession.send_message_async",
+ send_message_async("vertexai.send_message", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "vertexai.language_models",
+ "TextGenerationModel.predict_async",
+ predict_async("vertexai.predict", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "vertexai.language_models",
+ "TextGenerationModel.predict_streaming_async",
+ predict_streaming_async("vertexai.predict", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "vertexai.language_models",
+ "ChatSession.send_message_async",
+ start_chat_async("vertexai.send_message", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "vertexai.language_models",
+ "ChatSession.send_message_streaming_async",
+ start_chat_streaming_async("vertexai.send_message", version, environment,
+ application_name, tracer, pricing_info, trace_content,
+ metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "vertexai.language_models",
+ "TextEmbeddingModel.get_embeddings_async",
+ embeddings_async("vertexai.get_embeddings", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ def _uninstrument(self, **kwargs):
+ # Proper uninstrumentation logic to revert patched methods
+ pass
diff --git a/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py b/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py
new file mode 100644
index 00000000..91ec4bd7
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py
@@ -0,0 +1,1047 @@
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, protected-access, too-many-lines
+"""
+Module for monitoring VertexAI API calls.
+"""
+
+import logging
+import math
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
+from openlit.__helpers import get_chat_model_cost, get_embed_model_cost, handle_exception
+from openlit.semcov import SemanticConvetion
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+def generate_content_async(gen_ai_endpoint, version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for messages to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'generate_content' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'generate_content' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'generate_content' method.
+ kwargs: Keyword arguments for the 'generate_content' method.
+
+ Returns:
+ The response from the original 'generate_content' method.
+ """
+
+ # Check if streaming is enabled for the API call
+ streaming = kwargs.get("stream", False)
+
+ # pylint: disable=no-else-return
+ if streaming:
+ # Special handling for streaming response to accommodate the nature of data flow
+ async def stream_generator():
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ async for event in await wrapped(*args, **kwargs):
+ llmresponse += str(event.text)
+ prompt_tokens = event.usage_metadata.prompt_token_count
+ completion_tokens = event.usage_metadata.candidates_token_count
+ total_tokens = event.usage_metadata.total_token_count
+ yield event
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ prompt = str(args[0][0])
+
+ model = "/".join(instance._model_name.split("/")[3:])
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info, prompt_tokens,
+ completion_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ total_tokens, attributes
+ )
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ # Handling for non-streaming responses
+ else:
+ with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:
+ response = await wrapped(*args, **kwargs)
+
+ try:
+ # Format 'messages' into a single string
+ prompt = str(args[0][0])
+
+ model = "/".join(instance._model_name.split("/")[3:])
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info,
+ response.usage_metadata.prompt_token_count,
+ response.usage_metadata.candidates_token_count)
+
+ # Set Span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage_metadata.prompt_token_count)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.usage_metadata.candidates_token_count)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage_metadata.total_token_count)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ # pylint: disable=line-too-long
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION, response.candidates[0].content.parts[0].text)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ response.usage_metadata.total_token_count, attributes)
+ metrics["genai_completion_tokens"].add(
+ response.usage_metadata.candidates_token_count, attributes)
+ metrics["genai_prompt_tokens"].add(
+ response.usage_metadata.prompt_token_count, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+
+def send_message_async(gen_ai_endpoint, version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for messages to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'generate_content' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'generate_content' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'generate_content' method.
+ kwargs: Keyword arguments for the 'generate_content' method.
+
+ Returns:
+ The response from the original 'generate_content' method.
+ """
+
+ # Check if streaming is enabled for the API call
+ streaming = kwargs.get("stream", False)
+
+ # pylint: disable=no-else-return
+ if streaming:
+ # Special handling for streaming response to accommodate the nature of data flow
+ async def stream_generator():
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ async for event in await wrapped(*args, **kwargs):
+ llmresponse += str(event.text)
+ prompt_tokens = event.usage_metadata.prompt_token_count
+ completion_tokens = event.usage_metadata.candidates_token_count
+ total_tokens = event.usage_metadata.total_token_count
+ yield event
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ prompt = args[0][0]
+
+ model = "/".join(instance._model._model_name.split("/")[3:])
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info, prompt_tokens,
+ completion_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ total_tokens, attributes
+ )
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ # Handling for non-streaming responses
+ else:
+ with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:
+ response = await wrapped(*args, **kwargs)
+
+ try:
+ # Format 'messages' into a single string
+ prompt = args[0]
+
+ model = "/".join(instance._model._model_name.split("/")[3:])
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info,
+ response.usage_metadata.prompt_token_count,
+ response.usage_metadata.candidates_token_count)
+
+ # Set Span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage_metadata.prompt_token_count)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.usage_metadata.candidates_token_count)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage_metadata.total_token_count)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ # pylint: disable=line-too-long
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ response.candidates[0].content.parts[0].text)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ response.usage_metadata.total_token_count, attributes)
+ metrics["genai_completion_tokens"].add(
+ response.usage_metadata.candidates_token_count, attributes)
+ metrics["genai_prompt_tokens"].add(
+ response.usage_metadata.prompt_token_count, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def predict_async(gen_ai_endpoint, version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for messages to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'generate_content' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'generate_content' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'generate_content' method.
+ kwargs: Keyword arguments for the 'generate_content' method.
+
+ Returns:
+ The response from the original 'generate_content' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:
+ response = await wrapped(*args, **kwargs)
+
+ try:
+ prompt = args[0]
+
+ model = instance._model_id
+ #pylint: disable=line-too-long
+ prompt_tokens = response._prediction_response.metadata["tokenMetadata"]["inputTokenCount"]["totalTokens"]
+ completion_tokens = response._prediction_response.metadata["tokenMetadata"]["outputTokenCount"]["totalTokens"]
+ total_tokens = prompt_tokens + completion_tokens
+
+ #Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info, prompt_tokens,
+ completion_tokens)
+
+ # Set Span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ # pylint: disable=line-too-long
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION, response.text)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(
+ completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(
+ prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def predict_streaming_async(gen_ai_endpoint, version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for messages to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'predict' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'predict' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'predict' method.
+ kwargs: Keyword arguments for the 'predict' method.
+
+ Returns:
+ The response from the original 'predict' method.
+ """
+
+ # Special handling for streaming response to accommodate the nature of data flow
+ async def stream_generator():
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ async for event in wrapped(*args, **kwargs):
+ llmresponse += str(event)
+ yield event
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ prompt = args[0]
+ llmresponse = llmresponse.split('TextGenerationResponse',
+ maxsplit=1)[0].rstrip()
+
+ prompt_tokens = math.ceil(len(prompt) / 4)
+ completion_tokens = math.ceil(len(llmresponse) / 4)
+ total_tokens = prompt_tokens + completion_tokens
+
+ model = instance._model_id
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info, prompt_tokens,
+ completion_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ total_tokens, attributes
+ )
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ return wrapper
+
+def start_chat_async(gen_ai_endpoint, version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for messages to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'generate_content' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'generate_content' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'generate_content' method.
+ kwargs: Keyword arguments for the 'generate_content' method.
+
+ Returns:
+ The response from the original 'generate_content' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:
+ response = await wrapped(*args, **kwargs)
+
+ try:
+ prompt = args[0]
+
+ model = instance._model._model_id
+
+ #pylint: disable=line-too-long
+ prompt_tokens = response._prediction_response.metadata["tokenMetadata"]["inputTokenCount"]["totalTokens"]
+ completion_tokens = response._prediction_response.metadata["tokenMetadata"]["outputTokenCount"]["totalTokens"]
+ total_tokens = prompt_tokens + completion_tokens
+
+ #Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info, prompt_tokens,
+ completion_tokens)
+
+ # Set Span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ # pylint: disable=line-too-long
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION, response.text)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(
+ completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(
+ prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def start_chat_streaming_async(gen_ai_endpoint, version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for messages to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'start_chat' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'start_chat' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'start_chat' method.
+ kwargs: Keyword arguments for the 'start_chat' method.
+
+ Returns:
+ The response from the original 'start_chat' method.
+ """
+
+ # Special handling for streaming response to accommodate the nature of data flow
+ async def stream_generator():
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ async for event in wrapped(*args, **kwargs):
+ llmresponse += str(event.text)
+ yield event
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ prompt = args[0]
+
+ prompt_tokens = math.ceil(len(prompt) / 4)
+ completion_tokens = math.ceil(len(llmresponse) / 4)
+ total_tokens = prompt_tokens + completion_tokens
+
+ model = instance._model._model_id
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info, prompt_tokens,
+ completion_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ total_tokens, attributes
+ )
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ return wrapper
+
+def embeddings_async(gen_ai_endpoint, version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for messages to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'generate_content' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'generate_content' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'generate_content' method.
+ kwargs: Keyword arguments for the 'generate_content' method.
+
+ Returns:
+ The response from the original 'generate_content' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:
+ response = await wrapped(*args, **kwargs)
+
+ try:
+ prompt = args[0][0]
+
+ model = instance._model_id
+
+ prompt_tokens = int(response[0].statistics.token_count)
+
+ #Calculate cost of the operation
+ cost = get_embed_model_cost(model,
+ pricing_info, prompt_tokens)
+
+ # Set Span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ prompt_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(
+ prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/vertexai/vertexai.py b/sdk/python/src/openlit/instrumentation/vertexai/vertexai.py
new file mode 100644
index 00000000..2657dfae
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/vertexai/vertexai.py
@@ -0,0 +1,1047 @@
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, protected-access, too-many-lines
+"""
+Module for monitoring VertexAI API calls.
+"""
+
+import logging
+import math
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
+from openlit.__helpers import get_chat_model_cost, get_embed_model_cost, handle_exception
+from openlit.semcov import SemanticConvetion
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+def generate_content(gen_ai_endpoint, version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for messages to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'generate_content' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'generate_content' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'generate_content' method.
+ kwargs: Keyword arguments for the 'generate_content' method.
+
+ Returns:
+ The response from the original 'generate_content' method.
+ """
+
+ # Check if streaming is enabled for the API call
+ streaming = kwargs.get("stream", False)
+
+ # pylint: disable=no-else-return
+ if streaming:
+ # Special handling for streaming response to accommodate the nature of data flow
+ def stream_generator():
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ for event in wrapped(*args, **kwargs):
+ llmresponse += str(event.text)
+ prompt_tokens = event.usage_metadata.prompt_token_count
+ completion_tokens = event.usage_metadata.candidates_token_count
+ total_tokens = event.usage_metadata.total_token_count
+ yield event
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ prompt = str(args[0][0])
+
+ model = "/".join(instance._model_name.split("/")[3:])
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info, prompt_tokens,
+ completion_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ total_tokens, attributes
+ )
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ # Handling for non-streaming responses
+ else:
+ with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ # Format 'messages' into a single string
+ prompt = str(args[0][0])
+
+ model = "/".join(instance._model_name.split("/")[3:])
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info,
+ response.usage_metadata.prompt_token_count,
+ response.usage_metadata.candidates_token_count)
+
+ # Set Span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage_metadata.prompt_token_count)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.usage_metadata.candidates_token_count)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage_metadata.total_token_count)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ # pylint: disable=line-too-long
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION, response.candidates[0].content.parts[0].text)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ response.usage_metadata.total_token_count, attributes)
+ metrics["genai_completion_tokens"].add(
+ response.usage_metadata.candidates_token_count, attributes)
+ metrics["genai_prompt_tokens"].add(
+ response.usage_metadata.prompt_token_count, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+
+def send_message(gen_ai_endpoint, version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for messages to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'generate_content' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'generate_content' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'generate_content' method.
+ kwargs: Keyword arguments for the 'generate_content' method.
+
+ Returns:
+ The response from the original 'generate_content' method.
+ """
+
+ # Check if streaming is enabled for the API call
+ streaming = kwargs.get("stream", False)
+
+ # pylint: disable=no-else-return
+ if streaming:
+ # Special handling for streaming response to accommodate the nature of data flow
+ def stream_generator():
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ for event in wrapped(*args, **kwargs):
+ llmresponse += str(event.text)
+ prompt_tokens = event.usage_metadata.prompt_token_count
+ completion_tokens = event.usage_metadata.candidates_token_count
+ total_tokens = event.usage_metadata.total_token_count
+ yield event
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ prompt = args[0][0]
+
+ model = "/".join(instance._model._model_name.split("/")[3:])
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info, prompt_tokens,
+ completion_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ total_tokens, attributes
+ )
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ # Handling for non-streaming responses
+ else:
+ with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ # Format 'messages' into a single string
+ prompt = args[0]
+
+ model = "/".join(instance._model._model_name.split("/")[3:])
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info,
+ response.usage_metadata.prompt_token_count,
+ response.usage_metadata.candidates_token_count)
+
+ # Set Span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage_metadata.prompt_token_count)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.usage_metadata.candidates_token_count)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage_metadata.total_token_count)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ # pylint: disable=line-too-long
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ response.candidates[0].content.parts[0].text)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ response.usage_metadata.total_token_count, attributes)
+ metrics["genai_completion_tokens"].add(
+ response.usage_metadata.candidates_token_count, attributes)
+ metrics["genai_prompt_tokens"].add(
+ response.usage_metadata.prompt_token_count, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def predict(gen_ai_endpoint, version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for messages to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'predict' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'predict' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'predict' method.
+ kwargs: Keyword arguments for the 'predict' method.
+
+ Returns:
+ The response from the original 'predict' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ prompt = args[0]
+
+ model = instance._model_id
+ #pylint: disable=line-too-long
+ prompt_tokens = response._prediction_response.metadata["tokenMetadata"]["inputTokenCount"]["totalTokens"]
+ completion_tokens = response._prediction_response.metadata["tokenMetadata"]["outputTokenCount"]["totalTokens"]
+ total_tokens = prompt_tokens + completion_tokens
+
+ #Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info, prompt_tokens,
+ completion_tokens)
+
+ # Set Span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ # pylint: disable=line-too-long
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION, response.text)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(
+ completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(
+ prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def predict_streaming(gen_ai_endpoint, version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for messages to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'predict' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'predict' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'predict' method.
+ kwargs: Keyword arguments for the 'predict' method.
+
+ Returns:
+ The response from the original 'predict' method.
+ """
+
+ # Special handling for streaming response to accommodate the nature of data flow
+ def stream_generator():
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ for event in wrapped(*args, **kwargs):
+ llmresponse += str(event)
+ yield event
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ prompt = args[0]
+ llmresponse = llmresponse.split('TextGenerationResponse',
+ maxsplit=1)[0].rstrip()
+
+ prompt_tokens = math.ceil(len(prompt) / 4)
+ completion_tokens = math.ceil(len(llmresponse) / 4)
+ total_tokens = prompt_tokens + completion_tokens
+
+ model = instance._model_id
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info, prompt_tokens,
+ completion_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ total_tokens, attributes
+ )
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ return wrapper
+
+def start_chat(gen_ai_endpoint, version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for messages to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'start_chat' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'start_chat' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'start_chat' method.
+ kwargs: Keyword arguments for the 'start_chat' method.
+
+ Returns:
+ The response from the original 'start_chat' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ prompt = args[0]
+
+ model = instance._model._model_id
+
+ #pylint: disable=line-too-long
+ prompt_tokens = response._prediction_response.metadata["tokenMetadata"]["inputTokenCount"]["totalTokens"]
+ completion_tokens = response._prediction_response.metadata["tokenMetadata"]["outputTokenCount"]["totalTokens"]
+ total_tokens = prompt_tokens + completion_tokens
+
+ #Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info, prompt_tokens,
+ completion_tokens)
+
+ # Set Span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ # pylint: disable=line-too-long
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION, response.text)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(
+ completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(
+ prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def start_chat_streaming(gen_ai_endpoint, version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for messages to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'start_chat' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'start_chat' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'start_chat' method.
+ kwargs: Keyword arguments for the 'start_chat' method.
+
+ Returns:
+ The response from the original 'start_chat' method.
+ """
+
+ # Special handling for streaming response to accommodate the nature of data flow
+ def stream_generator():
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ for event in wrapped(*args, **kwargs):
+ llmresponse += str(event.text)
+ yield event
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ prompt = args[0]
+
+ prompt_tokens = math.ceil(len(prompt) / 4)
+ completion_tokens = math.ceil(len(llmresponse) / 4)
+ total_tokens = prompt_tokens + completion_tokens
+
+ model = instance._model._model_id
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info, prompt_tokens,
+ completion_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ total_tokens, attributes
+ )
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ return wrapper
+
+def embeddings(gen_ai_endpoint, version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for messages to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'generate_content' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'generate_content' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'generate_content' method.
+ kwargs: Keyword arguments for the 'generate_content' method.
+
+ Returns:
+ The response from the original 'generate_content' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ prompt = args[0][0]
+
+ model = instance._model_id
+
+ prompt_tokens = int(response[0].statistics.token_count)
+
+ #Calculate cost of the operation
+ cost = get_embed_model_cost(model,
+ pricing_info, prompt_tokens)
+
+ # Set Span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ prompt_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(
+ prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.py
index 2f12b23e..fcada45b 100644
--- a/sdk/python/src/openlit/semcov/__init__.py
+++ b/sdk/python/src/openlit/semcov/__init__.py
@@ -86,6 +86,7 @@ class SemanticConvetion:
GEN_AI_SYSTEM_COHERE = "cohere"
GEN_AI_SYSTEM_MISTRAL = "mistral"
GEN_AI_SYSTEM_BEDROCK = "bedrock"
+ GEN_AI_SYSTEM_VERTEXAI = "vertexai"
GEN_AI_SYSTEM_LANGCHAIN = "langchain"
# Vector DB
diff --git a/sdk/python/tests/requirements.txt b/sdk/python/tests/requirements.txt
index f8fc8983..a25d0900 100644
--- a/sdk/python/tests/requirements.txt
+++ b/sdk/python/tests/requirements.txt
@@ -12,6 +12,7 @@ mistralai>=0.1.0
transformers>=4.39.3
boto3>=1.34.93
botocore>=1.34.93
+google-cloud-aiplatform >= 1.38.1
# Vector DBs
chromadb>=0.4.0
diff --git a/sdk/python/tests/test_vertexai.py b/sdk/python/tests/test_vertexai.py
new file mode 100644
index 00000000..81cac5bf
--- /dev/null
+++ b/sdk/python/tests/test_vertexai.py
@@ -0,0 +1,382 @@
+# pylint: disable=duplicate-code, no-name-in-module, no-member, import-error
+"""
+This module contains tests for Vertex AI functionality using the Vertex AI Python library.
+
+Tests cover various API endpoints, including chat.
+These tests validate integration with OpenLIT.
+
+Environment Variables:
+ - GCP_PROJECT_ID: GCP Project ID for authentication.
+
+Note: Ensure the environment is properly configured for Vertex AI access and OpenLIT monitoring
+prior to running these tests.
+"""
+
+import os
+import pytest
+import vertexai
+from vertexai.generative_models import (
+ GenerativeModel
+)
+from vertexai.language_models import (
+ TextGenerationModel,
+ ChatModel,
+ TextEmbeddingModel
+)
+import openlit
+
+# Initialize Vertex AI
+vertexai.init(project=os.getenv("GCP_PROJECT_ID"), location="us-central1")
+
+
+# Initialize environment and application name for OpenLIT monitoring
+openlit.init(environment="openlit-testing", application_name="openlit-python-test")
+
+def test_sync_vertexai_send_message():
+ """
+ Tests synchronous send_message with the 'gemini-1.0-pro-001' model.
+
+ Raises:
+ AssertionError: If the send_message response object is not as expected.
+ """
+
+ try:
+ generation_config = {
+ "max_output_tokens": 15,
+ "temperature": 1,
+ "top_p": 1,
+ }
+ model = GenerativeModel(model_name="gemini-1.0-pro-001")
+ chat = model.start_chat()
+
+ response = chat.send_message("Just say 'LLM Observability'",
+ stream=False, generation_config=generation_config)
+
+ assert response.candidates[0].content.role == 'model'
+
+
+ works = False
+ responses = chat.send_message("Just say 'LLM Monitoring'",
+ stream=True, generation_config=generation_config)
+ for response in responses:
+ if response.candidates[0].content.role == 'model':
+ works = True
+
+ assert works is True
+
+ # pylint: disable=broad-exception-caught
+ except Exception as e:
+ if "rate limit" in str(e).lower():
+ print("Rate Limited:", e)
+
+def test_sync_vertexai_generate_content():
+ """
+ Tests synchronous generate_content with the 'gemini-1.0-pro-001' model.
+
+ Raises:
+ AssertionError: If the generate_content response object is not as expected.
+ """
+
+ try:
+ generation_config = {
+ "max_output_tokens": 25,
+ "temperature": 1,
+ "top_p": 1,
+ }
+ model = GenerativeModel("gemini-1.0-pro-002")
+
+ response = model.generate_content(
+ ["""Say 'LLM Observability'"""],
+ generation_config=generation_config,
+ stream=False,
+ )
+
+ assert response.candidates[0].content.role == 'model'
+
+
+ works = False
+ responses = model.generate_content(
+ ["""Say 'LLM Observability'"""],
+ generation_config=generation_config,
+ stream=True,
+ )
+ for response in responses:
+ if response.candidates[0].content.role == 'model':
+ works = True
+
+ assert works is True
+
+ # pylint: disable=broad-exception-caught
+ except Exception as e:
+ if "rate limit" in str(e).lower():
+ print("Rate Limited:", e)
+
+def test_sync_vertexai_predict():
+ """
+ Tests synchronous predict with the text-biason' model.
+
+ Raises:
+ AssertionError: If the predict response object is not as expected.
+ """
+
+ try:
+ parameters = {
+ "max_output_tokens": 1,
+ }
+ model = TextGenerationModel.from_pretrained("text-bison@002")
+
+ response = model.predict(
+ "Just say 'LLM Observability'",
+ **parameters,
+ )
+
+ assert isinstance(response.text, str)
+
+ works = False
+ responses = model.predict_streaming(
+ "Just say 'LLM Observability'",
+ **parameters,
+ )
+ for response in responses:
+ if response.is_blocked is False:
+ works = True
+
+ assert works is True
+
+ # pylint: disable=broad-exception-caught
+ except Exception as e:
+ if "rate limit" in str(e).lower():
+ print("Rate Limited:", e)
+
+def test_sync_vertexai_start_chat():
+ """
+ Tests synchronous start_chat with the chat-biason' model.
+
+ Raises:
+ AssertionError: If the start_chat response object is not as expected.
+ """
+
+ try:
+ parameters = {
+ "max_output_tokens": 1,
+ }
+ chat_model = ChatModel.from_pretrained("chat-bison@002")
+ chat = chat_model.start_chat(
+ context="LLM Monitoring",
+ )
+
+ response = chat.send_message(
+ "What is LLM Observability?", **parameters
+ )
+
+ assert isinstance(response.text, str)
+
+ works = False
+ responses = chat.send_message_streaming(
+ "What is LLM Observability?", **parameters
+ )
+ for response in responses:
+ if response.is_blocked is False:
+ works = True
+
+ assert works is True
+
+ # pylint: disable=broad-exception-caught
+ except Exception as e:
+ if "rate limit" in str(e).lower():
+ print("Rate Limited:", e)
+
+def test_sync_vertexai_get_embeddings():
+ """
+ Tests synchronous get_embeddings with the textembedding-gecko@001 model.
+
+ Raises:
+ AssertionError: If the get_embeddings response object is not as expected.
+ """
+
+ try:
+ model = TextEmbeddingModel.from_pretrained("textembedding-gecko@001")
+ response = model.get_embeddings(["What is LLM Observability?"])
+
+ assert response[0].statistics.truncated is False
+
+ # pylint: disable=broad-exception-caught
+ except Exception as e:
+ if "rate limit" in str(e).lower():
+ print("Rate Limited:", e)
+
+async def test_async_vertexai_send_message():
+ """
+ Tests asynchronous send_message with the 'gemini-1.0-pro-001' model.
+
+ Raises:
+ AssertionError: If the send_message response object is not as expected.
+ """
+
+ try:
+ generation_config = {
+ "max_output_tokens": 15,
+ "temperature": 1,
+ "top_p": 1,
+ }
+ model = GenerativeModel(model_name="gemini-1.0-pro-001")
+ chat = model.start_chat()
+
+ response = await chat.send_message_async("Just say 'LLM Observability'",
+ stream=False, generation_config=generation_config)
+
+ assert response.candidates[0].content.role == 'model'
+
+ works = False
+ responses = await chat.send_message_async("Just say 'LLM Observability'",
+ stream=True, generation_config=generation_config)
+ async for response in responses:
+ if response.candidates[0].content.role == 'model':
+ works = True
+
+ assert works is True
+
+ # pylint: disable=broad-exception-caught
+ except Exception as e:
+ if "rate limit" in str(e).lower():
+ print("Rate Limited:", e)
+
+async def test_async_vertexai_generate_content():
+ """
+ Tests asynchronous generate_content with the 'gemini-1.0-pro-001' model.
+
+ Raises:
+ AssertionError: If the generate_content response object is not as expected.
+ """
+
+ try:
+ generation_config = {
+ "max_output_tokens": 25,
+ "temperature": 1,
+ "top_p": 1,
+ }
+ model = GenerativeModel("gemini-1.0-pro-002")
+
+ response = await model.generate_content_async(
+ ["""Say 'LLM Observability'"""],
+ generation_config=generation_config,
+ stream=False,
+ )
+
+ assert response.candidates[0].content.role == 'model'
+
+
+ works = False
+ responses = await model.generate_content_async(
+ ["""Say 'LLM Observability'"""],
+ generation_config=generation_config,
+ stream=True,
+ )
+ for response in responses:
+ if response.candidates[0].content.role == 'model':
+ works = True
+
+ assert works is True
+
+ # pylint: disable=broad-exception-caught
+ except Exception as e:
+ if "rate limit" in str(e).lower():
+ print("Rate Limited:", e)
+
+async def test_async_vertexai_predict():
+ """
+ Tests asynchronous predict with the text-biason' model.
+
+ Raises:
+ AssertionError: If the predict response object is not as expected.
+ """
+
+ try:
+ parameters = {
+ "max_output_tokens": 1,
+ }
+ model = TextGenerationModel.from_pretrained("text-bison@002")
+ response = await model.predict_async(
+ "Just say 'LLM Observability'",
+ **parameters,
+ )
+
+ assert isinstance(response.text, str)
+
+ works = False
+ responses = await model.predict_streaming_async(
+ "Just say 'LLM Observability'",
+ **parameters,
+ )
+ async for response in responses:
+ if response.is_blocked is False:
+ works = True
+
+ assert works is True
+
+ # pylint: disable=broad-exception-caught
+ except Exception as e:
+ if "rate limit" in str(e).lower():
+ print("Rate Limited:", e)
+
+async def test_async_vertexai_start_chat():
+ """
+ Tests asynchronous start_chat with the chat-biason' model.
+
+ Raises:
+ AssertionError: If the start_chat response object is not as expected.
+ """
+
+ try:
+ parameters = {
+ "max_output_tokens": 1,
+ }
+ chat_model = ChatModel.from_pretrained("chat-bison@002")
+ chat = chat_model.start_chat(
+ context="LLM Monitoring",
+ )
+
+ response = await chat.send_message_async(
+ "What is LLM Observability?", **parameters
+ )
+
+ assert isinstance(response.text, str)
+
+ works = False
+ responses = await chat.send_message_streaming_async(
+ "What is LLM Observability?", **parameters
+ )
+ async for response in responses:
+ if response.is_blocked is False:
+ works = True
+
+ assert works is True
+
+ # pylint: disable=broad-exception-caught
+ except Exception as e:
+ if "rate limit" in str(e).lower():
+ print("Rate Limited:", e)
+
+async def test_async_vertexai_get_embeddings():
+ """
+ Tests synchronous get_embeddings with the textembedding-gecko@001 model.
+
+ Raises:
+ AssertionError: If the get_embeddings response object is not as expected.
+ """
+
+ try:
+ model = TextEmbeddingModel.from_pretrained("textembedding-gecko@001")
+ response = await model.get_embeddings_async(["What is LLM Observability?"])
+
+ assert response[0].statistics.truncated is False
+
+ # pylint: disable=broad-exception-caught
+ except Exception as e:
+ if "rate limit" in str(e).lower():
+ print("Rate Limited:", e)