<?xml version="1.0"?>

<ownershipDocument>

    <documentType>4</documentType>

    <periodOfReport>2003-07-01</periodOfReport>


    <notSubjectToSection16>0</notSubjectToSection16>

    <issuer>
        <issuerCik>0000813298</issuerCik>
        <issuerName>CASUAL MALE RETAIL GROUP INC</issuerName>
        <issuerTradingSymbol>CMRG</issuerTradingSymbol>
    </issuer>

    <reportingOwner>
        <reportingOwnerId>
            <rptOwnerCik>0000904439</rptOwnerCik>
            <rptOwnerName>HOLTZMAN SEYMOUR</rptOwnerName>
        </reportingOwnerId>
        <reportingOwnerAddress>
            <rptOwnerStreet1>  </rptOwnerStreet1>
            <rptOwnerStreet2>  </rptOwnerStreet2>
            <rptOwnerCity>  </rptOwnerCity>
            <rptOwnerState>  </rptOwnerState>
            <rptOwnerZipCode>  </rptOwnerZipCode>
        </reportingOwnerAddress>
        <reportingOwnerRelationship>
            <isDirector>1</isDirector>
            <isOfficer>0</isOfficer>
            <isTenPercentOwner>0</isTenPercentOwner>
            <isOther>0</isOther>
        </reportingOwnerRelationship>
    </reportingOwner>

    <nonDerivativeSecurity>
        <securityTitle>
            <value>Common Stock, $.01 par value</value>
        </securityTitle>
        <transactionDate>
            <value>2003-07-01</value>
        </transactionDate>
        <transactionCoding>
            <transactionFormType>4</transactionFormType>
            <transactionCode>A</transactionCode>
            <equitySwapInvolved>0</equitySwapInvolved>
        </transactionCoding>
        <transactionTimeliness>
            <value></value>
        </transactionTimeliness>
        <transactionAmounts>
            <transactionShares>
                <value>12820</value>
            </transactionShares>
            <transactionValue>
                <value>3.9</value>
                <footnoteId id="F1"/>
            </transactionValue>
            <transactionAcquiredDisposedCode>
                <value>A</value>
            </transactionAcquiredDisposedCode>
        </transactionAmounts>
        <postTransactionAmounts>
            <sharesOwnedFollowingTransaction>
                <value>3896914</value>
            </sharesOwnedFollowingTransaction>
        </postTransactionAmounts>
        <ownershipNature>
            <directOrIndirectOwnership>
                <value>I</value>
                <footnoteId id="F2"/>
                <footnoteId id="F3"/>
            </directOrIndirectOwnership>
            <natureOfOwnership>
                <value>By Jewelcor Management, Inc.</value>
            </natureOfOwnership>
        </ownershipNature>
    </nonDerivativeSecurity>

    <derivativeSecurity>
        <securityTitle>
            <value>Employee Stock Option (right to buy)</value>
        </securityTitle>
        <conversionOrExercisePrice>
            <value>5.01</value>
        </conversionOrExercisePrice>
        <transactionDate>
            <value>2003-07-01</value>
        </transactionDate>
        <transactionCoding>
            <transactionFormType>4</transactionFormType>
            <transactionCode>A</transactionCode>
            <equitySwapInvolved>0</equitySwapInvolved>
        </transactionCoding>
        <transactionTimeliness>
            <value></value>
        </transactionTimeliness>
        <transactionAmounts>
            <transactionShares>
                <value>200000</value>
            </transactionShares>
            <transactionValue>
                <value>0</value>
            </transactionValue>
            <transactionAcquiredDisposedCode>
                <value>A</value>
            </transactionAcquiredDisposedCode>
        </transactionAmounts>
        <exerciseDate>
            <value>2003-07-01</value>
            <footnoteId id="F4"/>
        </exerciseDate>
        <expirationDate>
            <value>2013-07-01</value>
        </expirationDate>
        <underlyingSecurity>
            <underlyingSecurityTitle>
                <value>Common Stock</value>
            </underlyingSecurityTitle>
            <underlyingSecurityShares>
                <value>200000</value>
            </underlyingSecurityShares>
        </underlyingSecurity>
        <postTransactionAmounts>
            <sharesOwnedFollowingTransaction>
                <value>200000</value>
                <footnoteId id="F7"/>
            </sharesOwnedFollowingTransaction>
        </postTransactionAmounts>
        <ownershipNature>
            <directOrIndirectOwnership>
                <value>D</value>
            </directOrIndirectOwnership>
        </ownershipNature>
    </derivativeSecurity>

    <derivativeSecurity>
        <securityTitle>
            <value>Warrants (right to buy)</value>
        </securityTitle>
        <conversionOrExercisePrice>
            <value>4.76</value>
            <footnoteId id="F5"/>
        </conversionOrExercisePrice>
        <transactionDate>
            <value>2003-07-02</value>
        </transactionDate>
        <transactionCoding>
            <transactionFormType>4</transactionFormType>
            <transactionCode>A</transactionCode>
            <equitySwapInvolved>0</equitySwapInvolved>
        </transactionCoding>
        <transactionTimeliness>
            <value></value>
        </transactionTimeliness>
        <transactionAmounts>
            <transactionShares>
                <value>60000</value>
            </transactionShares>
            <transactionValue>
                <value>0</value>
            </transactionValue>
            <transactionAcquiredDisposedCode>
                <value>A</value>
            </transactionAcquiredDisposedCode>
        </transactionAmounts>
        <exerciseDate>
            <value>2003-07-02</value>
        </exerciseDate>
        <expirationDate>
            <value>2010-07-02</value>
        </expirationDate>
        <underlyingSecurity>
            <underlyingSecurityTitle>
                <value>Common Stock</value>
            </underlyingSecurityTitle>
            <underlyingSecurityShares>
                <value>60000</value>
            </underlyingSecurityShares>
        </underlyingSecurity>
        <postTransactionAmounts>
            <sharesOwnedFollowingTransaction>
                <value>60000</value>
                <footnoteId id="F6"/>
            </sharesOwnedFollowingTransaction>
        </postTransactionAmounts>
        <ownershipNature>
            <directOrIndirectOwnership>
                <value>I</value>
                <footnoteId id="F2"/>
            </directOrIndirectOwnership>
            <natureOfOwnership>
                <value>By Jewelcor Management, Inc.</value>
            </natureOfOwnership>
        </ownershipNature>
    </derivativeSecurity>

    <footnotes>
        <footnote id="F1">In connection with a certain consulting agreement with the Issuer, as extended, Jewelcor Management, Inc., of which the Reporting Person is an indirect controlling shareholder, received additional compensation of $50,000 payable in 12,820 shares of the Issuer's common stock, based on the $3.90 closing price of the common stock on April 28, 2003.</footnote>
        <footnote id="F2">The Reporting Person is an indirect controlling shareholder of Jewelcor Management, Inc., a holder of more than 10% of the Issuer's common stock.</footnote>
        <footnote id="F3">The Reporting Person has previously reported direct beneficial ownership of 12,765 shares of the Issuer's common stock.</footnote>
        <footnote id="F4">66,667 shares are currently exercisable and 66,666 shares will become exercisable on each of July 1, 2004 and July 1, 2005.</footnote>
        <footnote id="F5">Exercise price is based on the average closing price of the Issuer's common stock on the NASDAQ National Market for the period of 30 trading days ending July 1, 2003.</footnote>
        <footnote id="F6">The Reporting Person has previously reported issuance of a warrant to Jewelcor Management, Inc. to purchase 147,059 shares of the Issuer's common stock, which warrant is currently exercisable.</footnote>
        <footnote id="F7">The Reporting Person has previously reported grants of options to purchase an aggregate of 930,000 shares of the Issuer's common stock.</footnote>
    </footnotes>

    <ownerSignature>
        <signatureName>Arlene C. Feldman, Attorney-in-Fact for   Seymour Holtzman</signatureName>
        <signatureDate>2003-07-03</signatureDate>
    </ownerSignature>

</ownershipDocument>

