Updated pages and referrer filters to merge urls.

.eslintrc.json

@@ -14,6 +14,7 @@
   },
   "plugins": ["react"],
   "rules": {
+    "react/display-name": "off",
     "react/react-in-jsx-scope": "off",
     "react/prop-types": "off"
   },

README.md

@@ -6,7 +6,7 @@ Umami is a simple, fast, website analytics alternative to Google Analytics.
 
 A detailed getting started guide can be found at [https://umami.is/docs/](https://umami.is/docs/)
 
-## Installation from source
+## Installing from source
 
 ### Requirements
 
@@ -74,7 +74,7 @@ By default this will launch the application on `http://localhost:3000`. You will
 [proxy](https://docs.nginx.com/nginx/admin-guide/web-server/reverse-proxy/) requests from your web server
 or change the [port](https://nextjs.org/docs/api-reference/cli#production) to serve the application directly.
 
-## Installation with Docker
+## Installing with Docker
 
 To build the umami container and start up a Postgres database, run:
 

components/WebsiteDetails.js

@@ -1,7 +1,7 @@
 import React, { useEffect, useState } from 'react';
 import classNames from 'classnames';
 import WebsiteChart from 'components/metrics/WebsiteChart';
-import RankingsChart from 'components/metrics/RankingsChart';
+import MetricsTable from 'components/metrics/MetricsTable';
 import WorldMap from 'components/common/WorldMap';
 import Page from 'components/layout/Page';
 import WebsiteHeader from 'components/metrics/WebsiteHeader';
@@ -9,12 +9,14 @@ import MenuLayout from 'components/layout/MenuLayout';
 import Button from 'components/common/Button';
 import { getDateRange } from 'lib/date';
 import { get } from 'lib/web';
-import { browserFilter, urlFilter, refFilter, deviceFilter, countryFilter } from 'lib/filters';
 import Arrow from 'assets/arrow-right.svg';
 import styles from './WebsiteDetails.module.css';
-
-const pageviewClasses = 'col-md-12 col-lg-6';
-const sessionClasses = 'col-md-12 col-lg-4';
+import PagesTable from './metrics/PagesTable';
+import ReferrersTable from './metrics/ReferrersTable';
+import BrowsersTable from './metrics/BrowsersTable';
+import OSTable from './metrics/OSTable';
+import DevicesTable from './metrics/DevicesTable';
+import CountriesTable from './metrics/CountriesTable';
 
 export default function WebsiteDetails({ websiteId, defaultDateRange = '7day' }) {
   const [data, setData] = useState();
@@ -24,29 +26,30 @@ export default function WebsiteDetails({ websiteId, defaultDateRange = '7day' })
   const [expand, setExpand] = useState();
   const { startDate, endDate } = dateRange;
 
+  const BackButton = () => (
+    <Button
+      className={styles.backButton}
+      icon={<Arrow />}
+      size="xsmall"
+      onClick={() => setExpand(null)}
+    >
+      <div>Back</div>
+    </Button>
+  );
+
   const menuOptions = [
     {
-      render: () => (
-        <Button
-          className={styles.backButton}
-          icon={<Arrow />}
-          size="xsmall"
-          onClick={() => setExpand(null)}
-        >
-          <div>Back</div>
-        </Button>
-      ),
+      render: BackButton,
     },
-    { label: 'Pages', value: 'url', filter: urlFilter },
-    { label: 'Referrers', value: 'referrer', filter: refFilter(data?.domain) },
-    { label: 'Browsers', value: 'browser', filter: browserFilter },
-    { label: 'Operating system', value: 'os' },
-    { label: 'Devices', value: 'device', filter: deviceFilter },
+    { label: 'Pages', value: 'url', component: PagesTable },
+    { label: 'Referrers', value: 'referrer', component: ReferrersTable },
+    { label: 'Browsers', value: 'browser', component: BrowsersTable },
+    { label: 'Operating system', value: 'os', component: OSTable },
+    { label: 'Devices', value: 'device', component: DevicesTable },
     {
       label: 'Countries',
       value: 'country',
-      filter: countryFilter,
-      onDataLoad: data => setCountryData(data),
+      component: props => <CountriesTable {...props} onDataLoad={data => setCountryData(data)} />,
     },
   ];
 
@@ -70,7 +73,7 @@ export default function WebsiteDetails({ websiteId, defaultDateRange = '7day' })
     setExpand(menuOptions.find(e => e.value === value));
   }
 
-  function getHeading(type) {
+  function getMetricLabel(type) {
     return type === 'url' || type === 'referrer' ? 'Views' : 'Visitors';
   }
 
@@ -84,6 +87,15 @@ export default function WebsiteDetails({ websiteId, defaultDateRange = '7day' })
     return null;
   }
 
+  const tableProps = {
+    websiteId,
+    startDate,
+    endDate,
+    limit: 10,
+    onExpand: handleExpand,
+    websiteDomain: data?.domain,
+  };
+
   return (
     <Page>
       <div className="row">
@@ -100,71 +112,22 @@ export default function WebsiteDetails({ websiteId, defaultDateRange = '7day' })
       {chartLoaded && !expand && (
         <>
           <div className={classNames(styles.row, 'row')}>
-            <div className={pageviewClasses}>
-              <RankingsChart
-                title="Pages"
-                type="url"
-                heading="Views"
-                websiteId={websiteId}
-                startDate={startDate}
-                endDate={endDate}
-                limit={10}
-                dataFilter={urlFilter}
-                onExpand={handleExpand}
-              />
+            <div className="col-md-12 col-lg-6">
+              <PagesTable {...tableProps} />
             </div>
-            <div className={pageviewClasses}>
-              <RankingsChart
-                title="Referrers"
-                type="referrer"
-                heading="Views"
-                websiteId={websiteId}
-                startDate={startDate}
-                endDate={endDate}
-                limit={10}
-                dataFilter={refFilter(data.domain)}
-                onExpand={handleExpand}
-              />
+            <div className="col-md-12 col-lg-6">
+              <ReferrersTable {...tableProps} />
             </div>
           </div>
           <div className={classNames(styles.row, 'row')}>
-            <div className={sessionClasses}>
-              <RankingsChart
-                title="Browsers"
-                type="browser"
-                heading="Visitors"
-                websiteId={websiteId}
-                startDate={startDate}
-                endDate={endDate}
-                limit={10}
-                dataFilter={browserFilter}
-                onExpand={handleExpand}
-              />
+            <div className="col-md-12 col-lg-4">
+              <BrowsersTable {...tableProps} />
             </div>
-            <div className={sessionClasses}>
-              <RankingsChart
-                title="Operating system"
-                type="os"
-                heading="Visitors"
-                websiteId={websiteId}
-                startDate={startDate}
-                endDate={endDate}
-                limit={10}
-                onExpand={handleExpand}
-              />
+            <div className="col-md-12 col-lg-4">
+              <OSTable {...tableProps} />
             </div>
-            <div className={sessionClasses}>
-              <RankingsChart
-                title="Devices"
-                type="device"
-                heading="Visitors"
-                websiteId={websiteId}
-                startDate={startDate}
-                endDate={endDate}
-                limit={10}
-                dataFilter={deviceFilter}
-                onExpand={handleExpand}
-              />
+            <div className="col-md-12 col-lg-4">
+              <DevicesTable {...tableProps} />
             </div>
           </div>
           <div className={classNames(styles.row, 'row')}>
@@ -172,18 +135,7 @@ export default function WebsiteDetails({ websiteId, defaultDateRange = '7day' })
               <WorldMap data={countryData} />
             </div>
             <div className="col-12 col-md-12 col-lg-4">
-              <RankingsChart
-                title="Countries"
-                type="country"
-                heading="Visitors"
-                websiteId={websiteId}
-                startDate={startDate}
-                endDate={endDate}
-                limit={10}
-                dataFilter={countryFilter}
-                onDataLoad={data => setCountryData(data)}
-                onExpand={handleExpand}
-              />
+              <CountriesTable {...tableProps} onDataLoad={data => setCountryData(data)} />
             </div>
           </div>
         </>
@@ -197,16 +149,7 @@ export default function WebsiteDetails({ websiteId, defaultDateRange = '7day' })
           selectedOption={expand.value}
           onMenuSelect={handleMenuSelect}
         >
-          <RankingsChart
-            title={expand.label}
-            type={expand.value}
-            heading={getHeading(expand.value)}
-            websiteId={websiteId}
-            startDate={startDate}
-            endDate={endDate}
-            dataFilter={expand.filter}
-            onDataLoad={expand.onDataLoad}
-          />
+          {expand.component({ ...tableProps, limit: false })}
         </MenuLayout>
       )}
     </Page>

components/common/Button.module.css

@@ -17,10 +17,6 @@
   background: #eaeaea;
 }
 
-.button + .button {
-  margin-left: 10px;
-}
-
 .large {
   font-size: var(--font-size-large);
 }

components/layout/ButtonLayout.js

@@ -0,0 +1,7 @@
+import React from 'react';
+import classNames from 'classnames';
+import styles from './ButtonLayout.module.css';
+
+export default function ButtonLayout({ className, children }) {
+  return <div className={classNames(styles.buttons, className)}>{children}</div>;
+}

components/layout/ButtonLayout.module.css

@@ -0,0 +1,7 @@
+.buttons {
+  display: flex;
+}
+
+.buttons button + button {
+  margin-left: 10px;
+}

components/layout/FormLayout.module.css

@@ -23,6 +23,10 @@
   margin-top: 20px;
 }
 
+.buttons button + button {
+  margin-left: 10px;
+}
+
 .error {
   position: absolute;
   display: flex;

components/metrics/BrowsersTable.js

@@ -0,0 +1,19 @@
+import React from 'react';
+import MetricsTable from './MetricsTable';
+import { browserFilter } from 'lib/filters';
+
+export default function BrowsersTable({ websiteId, startDate, endDate, limit, onExpand }) {
+  return (
+    <MetricsTable
+      title="Browsers"
+      type="browser"
+      metric="Visitors"
+      websiteId={websiteId}
+      startDate={startDate}
+      endDate={endDate}
+      limit={limit}
+      dataFilter={browserFilter}
+      onExpand={onExpand}
+    />
+  );
+}

components/metrics/CountriesTable.js

@@ -0,0 +1,27 @@
+import React from 'react';
+import MetricsTable from './MetricsTable';
+import { countryFilter, percentFilter } from 'lib/filters';
+
+export default function CountriesTable({
+  websiteId,
+  startDate,
+  endDate,
+  limit,
+  onDataLoad,
+  onExpand,
+}) {
+  return (
+    <MetricsTable
+      title="Countries"
+      type="country"
+      metric="Visitors"
+      websiteId={websiteId}
+      startDate={startDate}
+      endDate={endDate}
+      limit={limit}
+      dataFilter={countryFilter}
+      onDataLoad={data => onDataLoad(percentFilter(data))}
+      onExpand={onExpand}
+    />
+  );
+}

components/metrics/DevicesTable.js

@@ -0,0 +1,19 @@
+import React from 'react';
+import MetricsTable from './MetricsTable';
+import { deviceFilter } from 'lib/filters';
+
+export default function DevicesTable({ websiteId, startDate, endDate, limit, onExpand }) {
+  return (
+    <MetricsTable
+      title="Devices"
+      type="device"
+      metric="Visitors"
+      websiteId={websiteId}
+      startDate={startDate}
+      endDate={endDate}
+      limit={limit}
+      dataFilter={deviceFilter}
+      onExpand={onExpand}
+    />
+  );
+}

components/metrics/MetricsTable.js

@@ -7,18 +7,20 @@ import Arrow from 'assets/arrow-right.svg';
 import { get } from 'lib/web';
 import { percentFilter } from 'lib/filters';
 import { formatNumber, formatLongNumber } from 'lib/format';
-import styles from './RankingsChart.module.css';
+import styles from './MetricsTable.module.css';
 
-export default function RankingsChart({
+export default function MetricsTable({
   title,
+  metric,
   websiteId,
   startDate,
   endDate,
   type,
-  heading,
   className,
   dataFilter,
+  filterOptions,
   limit,
+  headerComponent,
   onDataLoad = () => {},
   onExpand = () => {},
 }) {
@@ -29,7 +31,7 @@ export default function RankingsChart({
 
   const rankings = useMemo(() => {
     if (data) {
-      const items = dataFilter ? dataFilter(data) : data;
+      const items = percentFilter(dataFilter ? dataFilter(data, filterOptions) : data);
       if (limit) {
         return items.filter((e, i) => i < limit);
       }
@@ -45,10 +47,8 @@ export default function RankingsChart({
       type,
     });
 
-    const updated = percentFilter(data);
-
-    setData(updated);
-    onDataLoad(updated);
+    setData(data);
+    onDataLoad(data);
   }
 
   function handleSetFormat() {
@@ -88,8 +88,9 @@ export default function RankingsChart({
     <div className={classNames(styles.container, className)}>
       <div className={styles.header}>
         <div className={styles.title}>{title}</div>
-        <div className={styles.heading} onClick={handleSetFormat}>
-          {heading}
+        {headerComponent}
+        <div className={styles.metric} onClick={handleSetFormat}>
+          {metric}
         </div>
       </div>
       <div className={styles.body}>
@@ -121,9 +122,11 @@ const AnimatedRow = ({ label, value = 0, percent, animate, format, onClick }) =>
   });
 
   return (
-    <div className={styles.row} onClick={onClick}>
-      <div className={styles.label}>{label}</div>
-      <animated.div className={styles.value}>{props.y?.interpolate(format)}</animated.div>
+    <div className={styles.row}>
+      <div className={styles.label}>{decodeURI(label)}</div>
+      <div className={styles.value} onClick={onClick}>
+        <animated.div className={styles.value}>{props.y?.interpolate(format)}</animated.div>
+      </div>
       <div className={styles.percent}>
         <animated.div
           className={styles.bar}

components/metrics/MetricsTable.module.css

@@ -9,16 +9,18 @@
 
 .header {
   display: flex;
+  align-items: center;
+  justify-content: space-between;
   line-height: 40px;
 }
 
 .title {
-  flex: 1;
+  display: flex;
   font-weight: 600;
   font-size: var(--font-size-normal);
 }
 
-.heading {
+.metric {
   font-size: var(--font-size-small);
   text-align: center;
   width: 100px;

components/metrics/OSTable.js

@@ -0,0 +1,17 @@
+import React from 'react';
+import MetricsTable from './MetricsTable';
+
+export default function OSTable({ websiteId, startDate, endDate, limit, onExpand }) {
+  return (
+    <MetricsTable
+      title="Operating System"
+      type="os"
+      metric="Visitors"
+      websiteId={websiteId}
+      startDate={startDate}
+      endDate={endDate}
+      limit={limit}
+      onExpand={onExpand}
+    />
+  );
+}

components/metrics/PagesTable.js

@@ -0,0 +1,28 @@
+import React from 'react';
+import MetricsTable from './MetricsTable';
+import { urlFilter } from 'lib/filters';
+
+export default function PagesTable({
+  websiteId,
+  websiteDomain,
+  startDate,
+  endDate,
+  limit,
+  onExpand,
+}) {
+  return (
+    <MetricsTable
+      title="Pages"
+      type="url"
+      metric="Views"
+      headerComponent={null}
+      websiteId={websiteId}
+      startDate={startDate}
+      endDate={endDate}
+      limit={limit}
+      dataFilter={urlFilter}
+      filterOptions={{ domain: websiteDomain }}
+      onExpand={onExpand}
+    />
+  );
+}

components/metrics/ReferrersTable.js

@@ -0,0 +1,28 @@
+import React from 'react';
+import MetricsTable from './MetricsTable';
+import { refFilter } from 'lib/filters';
+
+export default function Referrers({
+  websiteId,
+  websiteDomain,
+  startDate,
+  endDate,
+  limit,
+  onExpand = () => {},
+}) {
+  return (
+    <MetricsTable
+      title="Referrers"
+      type="referrer"
+      metric="Views"
+      headerComponent={null}
+      websiteId={websiteId}
+      startDate={startDate}
+      endDate={endDate}
+      limit={limit}
+      dataFilter={refFilter}
+      filterOptions={{ domain: websiteDomain }}
+      onExpand={onExpand}
+    />
+  );
+}

components/settings/AccountSettings.js

@@ -6,6 +6,7 @@ import Icon from 'components/common/Icon';
 import Table from 'components/common/Table';
 import Modal from 'components/common/Modal';
 import AccountEditForm from 'components/forms/AccountEditForm';
+import ButtonLayout from 'components/layout/ButtonLayout';
 import Pen from 'assets/pen.svg';
 import Plus from 'assets/plus.svg';
 import Trash from 'assets/trash.svg';
@@ -25,14 +26,14 @@ export default function AccountSettings() {
 
   const Buttons = row =>
     row.username !== 'admin' ? (
-      <>
+      <ButtonLayout>
         <Button icon={<Pen />} size="small" onClick={() => setEditAccount(row)}>
           <div>Edit</div>
         </Button>
         <Button icon={<Trash />} size="small" onClick={() => setDeleteAccount(row)}>
           <div>Delete</div>
         </Button>
-      </>
+      </ButtonLayout>
     ) : null;
 
   const columns = [

components/settings/WebsiteSettings.js

@@ -9,6 +9,7 @@ import DeleteForm from '../forms/DeleteForm';
 import TrackingCodeForm from '../forms/TrackingCodeForm';
 import ShareUrlForm from '../forms/ShareUrlForm';
 import EmptyPlaceholder from 'components/common/EmptyPlaceholder';
+import ButtonLayout from 'components/layout/ButtonLayout';
 import Pen from 'assets/pen.svg';
 import Trash from 'assets/trash.svg';
 import Plus from 'assets/plus.svg';
@@ -27,7 +28,7 @@ export default function WebsiteSettings() {
   const [saved, setSaved] = useState(0);
 
   const Buttons = row => (
-    <>
+    <ButtonLayout>
       {row.share_id && (
         <Button
           icon={<Link />}
@@ -50,7 +51,7 @@ export default function WebsiteSettings() {
       <Button icon={<Trash />} size="small" onClick={() => setDeleteWebsite(row)}>
         <div>Delete</div>
       </Button>
-    </>
+    </ButtonLayout>
   );
 
   const columns = [

lib/filters.js

@@ -1,16 +1,108 @@
-import escape from 'escape-string-regexp';
+import firstBy from 'thenby';
 import { BROWSERS, ISO_COUNTRIES, DEVICES } from './constants';
+import { removeTrailingSlash } from './format';
 
 export const browserFilter = data =>
   data.map(({ x, ...props }) => ({ x: BROWSERS[x] || x, ...props }));
 
-export const urlFilter = data => data.filter(({ x }) => x !== '' && !x.startsWith('#'));
+export const urlFilter = (data, { domain }) => {
+  const isValidUrl = url => {
+    return url !== '' && !url.startsWith('#');
+  };
 
-export const refFilter = domain => data => {
-  const regex = new RegExp(escape(domain));
-  return data.filter(
-    ({ x }) => x !== '' && !x.startsWith('/') && !x.startsWith('#') && !regex.test(x),
-  );
+  const cleanUrl = url => {
+    try {
+      const { pathname, searchParams } = new URL(url);
+
+      const path = removeTrailingSlash(pathname);
+      const ref = searchParams.get('ref');
+      const query = ref ? `?ref=${ref}` : '';
+
+      return `${path}${query}`;
+    } catch {
+      return null;
+    }
+  };
+
+  const map = data.reduce((obj, { x, y }) => {
+    if (!isValidUrl(x)) {
+      return obj;
+    }
+
+    const url = cleanUrl(x.startsWith('/') ? `http://${domain}${x}` : x);
+
+    if (url) {
+      if (!obj[url]) {
+        obj[url] = y;
+      } else {
+        obj[url] += y;
+      }
+    }
+
+    return obj;
+  }, {});
+
+  return Object.keys(map)
+    .map(key => ({ x: key, y: map[key] }))
+    .sort(firstBy('y', -1).thenBy('x'));
+};
+
+export const refFilter = (data, { domain, domainsOnly }) => {
+  const isValidRef = ref => {
+    return ref !== '' && !ref.startsWith('/') && !ref.startsWith('#');
+  };
+
+  const cleanUrl = url => {
+    try {
+      const { hostname, origin, pathname, searchParams, protocol } = new URL(url);
+
+      if (hostname === domain) {
+        return null;
+      }
+
+      if (domainsOnly && hostname) {
+        return hostname;
+      }
+
+      if (!origin || origin === 'null') {
+        return `${protocol}${removeTrailingSlash(pathname)}`;
+      }
+
+      if (protocol.startsWith('http')) {
+        const path = removeTrailingSlash(pathname);
+        const ref = searchParams.get('ref');
+        const query = ref ? `?ref=${ref}` : '';
+
+        return `${origin}${path}${query}`;
+      }
+
+      return null;
+    } catch {
+      return null;
+    }
+  };
+
+  const map = data.reduce((obj, { x, y }) => {
+    if (!isValidRef(x)) {
+      return obj;
+    }
+
+    const url = cleanUrl(x);
+
+    if (url) {
+      if (!obj[url]) {
+        obj[url] = y;
+      } else {
+        obj[url] += y;
+      }
+    }
+
+    return obj;
+  }, {});
+
+  return Object.keys(map)
+    .map(key => ({ x: key, y: map[key] }))
+    .sort(firstBy('y', -1).thenBy('x'));
 };
 
 export const deviceFilter = data =>

lib/format.js

@@ -62,3 +62,7 @@ export function formatLongNumber(value) {
 
   return formatNumber(n);
 }
+
+export function removeTrailingSlash(url) {
+  return url.length > 1 && url.endsWith('/') ? url.slice(0, -1) : url;
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "umami",
-  "version": "0.14.0",
+  "version": "0.15.0",
   "description": "A simple, fast, website analytics alternative to Google Analytics. ",
   "author": "Mike Cao <mike@mikecao.com>",
   "license": "MIT",
@@ -68,6 +68,7 @@
     "redux": "^4.0.5",
     "redux-thunk": "^2.3.0",
     "request-ip": "^2.1.3",
+    "thenby": "^1.3.4",
     "tinycolor2": "^1.4.1",
     "unfetch": "^4.1.0",
     "uuid": "^8.3.0"

yarn.lock

@@ -8316,6 +8316,11 @@ text-table@^0.2.0:
   resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4"
   integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=
 
+thenby@^1.3.4:
+  version "1.3.4"
+  resolved "https://registry.yarnpkg.com/thenby/-/thenby-1.3.4.tgz#81581f6e1bb324c6dedeae9bfc28e59b1a2201cc"
+  integrity sha512-89Gi5raiWA3QZ4b2ePcEwswC3me9JIg+ToSgtE0JWeCynLnLxNr/f9G+xfo9K+Oj4AFdom8YNJjibIARTJmapQ==
+
 through2@^2.0.0:
   version "2.0.5"
   resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.5.tgz#01c1e39eb31d07cb7d03a96a70823260b23132cd"