Reply
New Member
Posts: 2
Registered: ‎11-12-2018
Accepted Solution

ERX having 5-minute downtime multiple times a day

Hello everyone,

    I never expected to actually have to post something here but I must. My EdgeRouter X has random 5-minute downtime multiple times per day and it appears to be due to ram usage. I have my ERX connected to my UNMS server. I notice in the logs that multiple times a day, it disconnects and reconnects. As it lasts so short, it is only possible to see exactly what happens in the one hour view of the graphs but I finally caught it a couple times so I think I know what is going on. It appears that just before the downtime, the ram usage is just above 50% (usually 53%). After this downtime, the ram usage drops. I am not sure if it restarts during this time or just freezes because I can not access my device, it is in a data center (probably not the best place for an ERX but I'm too cheap to upgrade). I have a feeling that this is due to NAT states. I run a public NTP server so I receive a large number of requests so it may be possible that I reach some form of state limit.

 

Any thoughts are greatly appreciated. Maybe there is a way to turn off connection tracking for specific NAT rules (note that I do not have a firewall enabled)

 

Here is my config with my UNMS key removed. An image of some recent outages is attached.

Spoiler
interfaces {
    ethernet eth0 {
        address dhcp
        description Management
        duplex auto
        speed auto
    }
    ethernet eth1 {
        address 23.129.32.83/27
        address 2602:fed2:fc0::1f/64
        description WAN
        duplex auto
        speed auto
    }
    ethernet eth2 {
        address 10.10.10.1/24
        address 2a0d:1a40:fa1::3/64
        description LAN
        duplex auto
        speed auto
    }
    ethernet eth3 {
        description EVIX
        duplex auto
        speed auto
    }
    ethernet eth4 {
        address 10.10.11.1/24
        duplex auto
        poe {
            output off
        }
        speed auto
    }
    loopback lo {
    }
    switch switch0 {
        mtu 1500
    }
    tunnel tun0 {
        address 2a05:dfc7:3e:201::3/64
        disable
        encapsulation gre
        local-ip 10.10.10.1
        multicast disable
        remote-ip 10.10.10.15
        ttl 255
    }
}
port-forward {
    auto-firewall enable
    hairpin-nat disable
    lan-interface eth2
    rule 1 {
        description "ssh for Xserve"
        forward-to {
            address 10.10.10.11
            port 22
        }
        original-port 23
        protocol tcp_udp
    }
    rule 2 {
        description "vns for Xserve"
        forward-to {
            address 10.10.10.11
            port 5900
        }
        original-port 5900
        protocol tcp_udp
    }
    rule 3 {
        description "switch web interface"
        forward-to {
            address 10.10.10.10
            port 80
        }
        original-port 81
        protocol tcp_udp
    }
    rule 4 {
        description "BGP Router"
        forward-to {
            address 10.10.10.15
            port 22
        }
        original-port 22
        protocol tcp_udp
    }
    rule 5 {
        description "SMB for Xserve"
        forward-to {
            address 10.10.10.11
            port 445
        }
        original-port 446
        protocol tcp_udp
    }
    rule 6 {
        description "Mosh for BGP Router"
        forward-to {
            address 10.10.10.15
        }
        original-port 60000-61000
        protocol tcp_udp
    }
    rule 7 {
        description NTP!
        forward-to {
            address 10.10.10.15
        }
        original-port 123
        protocol tcp_udp
    }
    rule 8 {
        description Teredo
        forward-to {
            address 10.10.10.15
        }
        original-port 3550
        protocol tcp_udp
    }
    rule 9 {
        description Teredo
        forward-to {
            address 10.10.10.15
        }
        original-port 3544
        protocol tcp_udp
    }
    wan-interface eth1
}
protocols {
    static {
        route 0.0.0.0/0 {
            next-hop 23.129.32.65 {
            }
        }
        route6 ::/0 {
            next-hop 2602:fed2:fc0::1 {
            }
        }
        route6 2a0d:1a40:fa0::/44 {
            next-hop 2a0d:1a40:fa1::1 {
            }
        }
        route6 2a06:e881:121::/48 {
            next-hop 2a06:e881:121:1b::1 {
            }
        }
    }
}
service {
    dhcp-server {
        disabled false
        hostfile-update disable
        shared-network-name LAN {
            authoritative enable
            description LAN
            subnet 10.10.10.0/24 {
                default-router 10.10.10.1
                dns-server 10.10.10.1
                dns-server 1.1.1.1
                domain-name thenetworknerds.ca
                lease 31536000
                start 10.10.10.15 {
                    stop 10.10.10.254
                }
                static-mapping bgp-router {
                    ip-address 10.10.10.15
                    mac-address 00:1e:52:f1:fd:cc
                }
            }
        }
        shared-network-name eth4 {
            authoritative disable
            subnet 10.10.11.0/24 {
                default-router 10.10.11.1
                dns-server 10.10.11.1
                dns-server 1.1.1.1
                lease 86400
                start 10.10.11.10 {
                    stop 10.10.11.254
                }
            }
        }
        static-arp disable
        use-dnsmasq disable
    }
    dns {
        forwarding {
            cache-size 100
            listen-on eth2
        }
    }
    gui {
        http-port 80
        https-port 443
        older-ciphers enable
    }
    lldp {
    }
    nat {
        rule 1 {
            destination {
                group {
                    address-group ADDRv4_eth1
                }
            }
            inbound-interface eth1
            inside-address {
                address 10.10.10.15
            }
            log disable
            protocol gre
            type destination
        }
        rule 5000 {
            description WAN
            log disable
            outbound-interface eth1
            protocol all
            source {
                address 10.10.10.0/24
            }
            type masquerade
        }
    }
    ssh {
        port 24
        protocol-version v2
    }
    unms {
        connection #### REMOVED ####
    }
}
system {
    flow-accounting {
        disable-memory-table
        ingress-capture post-dnat
        interface eth0
        netflow {
            enable-egress {
                engine-id 1
            }
            engine-id 0
            server 194.50.19.3 {
                port 2055
            }
            timeout {
                expiry-interval 60
                flow-generic 60
                icmp 60
                max-active-life 60
                tcp-fin 10
                tcp-generic 60
                tcp-rst 10
                udp 60
            }
            version 9
        }
        syslog-facility daemon
    }
    host-name edge00-fmt.fmt.thenetworknerds.ca
    login {
        user bryce {
            authentication {
                encrypted-password $6$DpuTZCU8NbK3Du$SUvydUfJmFsJxkpqfFeN36Gpk.wYVKO7xy3c58OKjc0kweNxuNI.rfYoyblTF5Hz9tlayFqqu5lMqChFUw/G71
                plaintext-password ""
            }
            full-name "Bryce Wilson"
            level admin
        }
    }
    name-server 1.1.1.1
    name-server 8.8.8.8
    name-server 1.0.0.1
    name-server 8.8.4.4
    ntp {
        server time00.thenetworknerds.ca {
            prefer
        }
        server time01.thenetworknerds.ca {
        }
    }
    offload {
        hwnat enable
        ipsec enable
    }
    syslog {
        global {
            facility all {
                level notice
            }
            facility protocols {
                level debug
            }
        }
    }
    time-zone America/Vancouver
    traffic-analysis {
        dpi enable
        export enable
    }
}


/* Warning: Do not remove the following line. */
/* === vyatta-config-version: "config-management@1:conntrack@1:cron@1:dhcp-relay@1:dhcp-server@4:firewall@5:ipsec@5:nat@3:qos@1:quagga@2:suspend@1:system@4:ubnt-pptp@1:ubnt-udapi-server@1:ubnt-unms@1:ubnt-util@1:vrrp@1:webgui@1:webproxy@1:zone-policy@1" === */
/* Release version: v2.0.0-beta.1.5132607.181017.1318 */

Screen Shot 2019-02-11 at 3.59.23 PM.png

Accepted Solutions
New Member
Posts: 1
Registered: ‎04-26-2018
Kudos: 1
Solutions: 1

Re: ERX having 5-minute downtime multiple times a day

It’s crashing because you have hwnat enabled and 2.0.0 doesn’t work correctly with hwnat enabled on the ER-X.  

View solution in original post


All Replies
New Member
Posts: 1
Registered: ‎04-26-2018
Kudos: 1
Solutions: 1

Re: ERX having 5-minute downtime multiple times a day

It’s crashing because you have hwnat enabled and 2.0.0 doesn’t work correctly with hwnat enabled on the ER-X.  

New Member
Posts: 2
Registered: ‎11-12-2018

Re: ERX having 5-minute downtime multiple times a day

Thank you very much! I guess I have not been searching for the right things. I don't need the full speed at the moment so I will disable it until it gets fixed.

Reply