<?xml version="1.0"?>
<ownershipDocument>

    <schemaVersion>X0202</schemaVersion>

    <documentType>4</documentType>

    <periodOfReport>2006-05-23</periodOfReport>

    <notSubjectToSection16>0</notSubjectToSection16>

    <issuer>
        <issuerCik>0000813298</issuerCik>
        <issuerName>CASUAL MALE RETAIL GROUP INC</issuerName>
        <issuerTradingSymbol>CMRG</issuerTradingSymbol>
    </issuer>

    <reportingOwner>
        <reportingOwnerId>
            <rptOwnerCik>0000904439</rptOwnerCik>
            <rptOwnerName>HOLTZMAN SEYMOUR</rptOwnerName>
        </reportingOwnerId>
        <reportingOwnerAddress>
            <rptOwnerStreet1>C/O JEWELCOR MANAGEMENT, INC.</rptOwnerStreet1>
            <rptOwnerStreet2>100 N. WILKES BARRE BLVD., 4TH FLOOR</rptOwnerStreet2>
            <rptOwnerCity>WILKES BARRE</rptOwnerCity>
            <rptOwnerState>PA</rptOwnerState>
            <rptOwnerZipCode>18702</rptOwnerZipCode>
            <rptOwnerStateDescription></rptOwnerStateDescription>
        </reportingOwnerAddress>
        <reportingOwnerRelationship>
            <isDirector>1</isDirector>
            <isOfficer>0</isOfficer>
            <isTenPercentOwner>1</isTenPercentOwner>
            <isOther>0</isOther>
            <officerTitle></officerTitle>
            <otherText></otherText>
        </reportingOwnerRelationship>
    </reportingOwner>

    <derivativeTable>
        <derivativeTransaction>
            <securityTitle>
                <value>Warrants (right to buy)</value>
            </securityTitle>
            <conversionOrExercisePrice>
                <value>5.26</value>
            </conversionOrExercisePrice>
            <transactionDate>
                <value>2006-05-23</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>J</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
                <footnoteId id="F1"/>
            </transactionCoding>
            <transactionTimeliness>
                <value></value>
            </transactionTimeliness>
            <transactionAmounts>
                <transactionShares>
                    <value>60000</value>
                </transactionShares>
                <transactionPricePerShare>
                    <value>382200</value>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>A</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <exerciseDate>
                <value>2003-07-02</value>
            </exerciseDate>
            <expirationDate>
                <value>2010-07-02</value>
            </expirationDate>
            <underlyingSecurity>
                <underlyingSecurityTitle>
                    <value>common stock</value>
                </underlyingSecurityTitle>
                <underlyingSecurityShares>
                    <value>60000</value>
                </underlyingSecurityShares>
            </underlyingSecurity>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>60000</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>D</value>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value></value>
                </natureOfOwnership>
            </ownershipNature>
        </derivativeTransaction>
        <derivativeTransaction>
            <securityTitle>
                <value>Warrants (right to buy)</value>
            </securityTitle>
            <conversionOrExercisePrice>
                <value>7.36</value>
            </conversionOrExercisePrice>
            <transactionDate>
                <value>2006-05-23</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>J</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
                <footnoteId id="F1"/>
            </transactionCoding>
            <transactionTimeliness>
                <value></value>
            </transactionTimeliness>
            <transactionAmounts>
                <transactionShares>
                    <value>20000</value>
                </transactionShares>
                <transactionPricePerShare>
                    <value>105200</value>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>A</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <exerciseDate>
                <value>2003-09-25</value>
            </exerciseDate>
            <expirationDate>
                <value>2010-07-02</value>
            </expirationDate>
            <underlyingSecurity>
                <underlyingSecurityTitle>
                    <value>common stock</value>
                </underlyingSecurityTitle>
                <underlyingSecurityShares>
                    <value>20000</value>
                </underlyingSecurityShares>
            </underlyingSecurity>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>20000</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>D</value>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value></value>
                </natureOfOwnership>
            </ownershipNature>
        </derivativeTransaction>
    </derivativeTable>

    <footnotes>
        <footnote id="F1">The warrants to purchase 80,000 shares of common stock of the Issuer that are the subject of this Form 4 (the &quot;Warrants&quot;) were previously reported as beneficially owned by the Reporting Person indirectly through Jewelcor Management, Inc., a corporation of which the Reporting Person is a controlling shareholder (&quot;Jewelcor&quot;).  This Form 4 reports the change in the form of beneficial ownership of the Warrants which occurred pursuant to the May 23, 2006 acquisition of direct ownership by the Reporting Person of such Warrants from Jewelcor.</footnote>
    </footnotes>

    <remarks></remarks>

    <ownerSignature>
        <signatureName>/s/ Seymour Holtzman</signatureName>
        <signatureDate>2006-05-25</signatureDate>
    </ownerSignature>
</ownershipDocument>
